00010000:00000010:2.1:1788201183.071750:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11d600. 00010000:00000010:2.1:1788201183.235402:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22700. 00010000:00000010:2.1:1788201183.235409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8e00. 00000080:00000010:2.1:1788201183.246135:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35f0000. 00000080:00000010:2.1:1788201183.246148:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201760. 00010000:00000010:2.1:1788201183.261786:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11d800. 00000080:00000010:2.1:1788201183.320157:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5461080. 00000080:00000010:2.1:1788201183.320174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00000080:00000010:2.1:1788201183.722070:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf439c0. 00000080:00000010:2.1:1788201183.722085:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:2.1:1788201183.724230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11d800. 00010000:00000010:2.1:1788201183.911811:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3cb00. 00010000:00000010:2.1:1788201183.911826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103200. 00010000:00000010:2.1:1788201184.263263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b600. 00010000:00000010:2.1:1788201184.263289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5200. 00010000:00000010:2.1:1788201184.892606:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab138ce00. 00010000:00000010:2.1:1788201184.892617:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11d000. 00010000:00000010:2.1:1788201185.284199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7000. 00010000:00000010:2.1:1788201185.284216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00000080:00000010:2.1:1788201185.297553:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7cb180. 00000080:00000010:2.1:1788201185.297565:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc00. 00010000:00000010:2.1:1788201185.301239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11d000. 00010000:00000010:2.1:1788201185.324828:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7d00. 00010000:00000010:2.1:1788201185.324835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00000080:00000010:2.1:1788201185.328900:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe15ac0. 00000080:00000010:2.1:1788201185.328920:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468d80. 00010000:00000010:2.1:1788201185.336250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11d000. 00000080:00000010:2.1:1788201186.557141:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5939c0. 00000080:00000010:2.1:1788201186.557151:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201d40. 00010000:00000010:2.1:1788201186.655370:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51900. 00010000:00000010:2.1:1788201186.655388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabba00. 00000080:00000010:2.1:1788201186.665433:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbdb0840. 00000080:00000010:2.1:1788201186.665453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:2.1:1788201186.674791:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11c800. 00010000:00000010:2.1:1788201186.807746:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51100. 00010000:00000010:2.1:1788201186.807764:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24600. 00010000:00000010:2.1:1788201186.816931:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540600. 00010000:00000010:2.1:1788201186.837776:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b300. 00010000:00000010:2.1:1788201186.837795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25a00. 00000080:00000010:2.1:1788201186.844339:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9a2940. 00000080:00000010:2.1:1788201186.844360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172920. 00010000:00000010:2.1:1788201186.852112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11c200. 00010000:00000010:2.1:1788201186.876233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b000. 00010000:00000010:2.1:1788201186.876247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24000. 00000080:00000010:2.1:1788201186.882701:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e5c200. 00000080:00000010:2.1:1788201186.882721:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686a0. 00010000:00000010:2.1:1788201186.887178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11c200. 00010000:00000010:2.1:1788201186.906316:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bc00. 00010000:00000010:2.1:1788201186.906331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4e00. 00000080:00000010:2.1:1788201186.922302:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8430c200. 00000080:00000010:2.1:1788201186.922322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468520. 00010000:00000010:2.1:1788201186.935641:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11c200. 00010000:00000010:2.1:1788201187.186727:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aadad5000. 00010000:00000010:2.1:1788201187.186741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5800. 00000080:00000010:2.1:1788201187.201208:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5634a40. 00000080:00000010:2.1:1788201187.201228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:2.1:1788201187.235162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038ec00. 00000080:00000010:2.1:1788201187.559077:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8ff380. 00000080:00000010:2.1:1788201187.559110:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468b40. 00010000:00000010:2.1:1788201187.561798:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038f200. 00010000:00000010:2.1:1788201187.770182:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aadad5d00. 00010000:00000010:2.1:1788201187.770191:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24a00. 00010000:00000010:2.1:1788201187.886109:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87196800. 00010000:00000010:2.1:1788201187.906208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aadad5e00. 00010000:00000010:2.1:1788201187.906224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2c2dc00. 00000080:00000010:2.1:1788201187.917844:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa953180. 00000080:00000010:2.1:1788201187.917873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c440. 00010000:00000010:2.1:1788201187.928139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941cc00. 00000080:00000010:2.1:1788201187.954515:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf9a940. 00000080:00000010:2.1:1788201187.954535:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c20. 00010000:00000010:2.1:1788201187.966145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941c000. 00010000:00000010:2.1:1788201188.002663:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2500. 00010000:00000010:2.1:1788201188.002672:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2c2c600. 00000080:00000010:2.1:1788201188.002681:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54be300. 00000080:00000010:2.1:1788201188.002687:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c60. 00010000:00000010:2.1:1788201188.183445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72200. 00010000:00000010:2.1:1788201188.183463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2c2d400. 00000080:00000010:2.1:1788201188.193151:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5cca40. 00000080:00000010:2.1:1788201188.193179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7c0. 00010000:00000010:2.1:1788201188.207405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941cc00. 00010000:00000010:2.1:1788201188.227365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72700. 00010000:00000010:2.1:1788201188.227377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2c2c000. 00000080:00000010:2.1:1788201188.239278:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcd2100. 00000080:00000010:2.1:1788201188.239297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894682c0. 00010000:00000010:2.1:1788201188.239310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941cc00. 00000080:00000010:2.1:1788201188.953545:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85964a40. 00000080:00000010:2.1:1788201188.953566:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:2.1:1788201189.457035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941c600. 00010000:00000010:2.1:1788201189.594267:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2d00. 00010000:00000010:2.1:1788201189.594286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe108800. 00010000:00000010:2.1:1788201190.188172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959d00. 00010000:00000010:2.1:1788201190.188189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11ce00. 00000080:00000010:2.1:1788201190.218159:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe46300. 00000080:00000010:2.1:1788201190.218171:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c020. 00010000:00000010:2.1:1788201190.994606:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aadad5000. 00010000:00000010:2.1:1788201190.994620:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9c00. 00000080:00000010:2.1:1788201191.486135:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe82100. 00000080:00000010:2.1:1788201191.486151:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cde0. 00010000:00000010:2.1:1788201191.492788:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217c00. 00010000:00000010:2.1:1788201191.507299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3d00. 00010000:00000010:2.1:1788201191.507313:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb318c000. 00000080:00000010:2.1:1788201191.518821:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37ab9c0. 00000080:00000010:2.1:1788201191.518841:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c80. 00010000:00000010:2.1:1788201191.530133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217c00. 00010000:00000010:2.1:1788201191.822238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab6e7ee00. 00010000:00000010:2.1:1788201191.822246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a200. 00010000:00000010:2.1:1788201192.230652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4600. 00010000:00000010:2.1:1788201192.230665:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e600. 00000080:00000010:2.1:1788201192.251352:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa364ca40. 00000080:00000010:2.1:1788201192.251366:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b60. 00010000:00000010:2.1:1788201192.337901:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4b00. 00010000:00000010:2.1:1788201192.337911:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e000. 00000080:00000010:2.1:1788201192.358595:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5667380. 00000080:00000010:2.1:1788201192.358616:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cca0. 00010000:00000010:2.1:1788201192.358627:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216a00. 00010000:00000010:2.1:1788201192.928176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab6e7e400. 00010000:00000010:2.1:1788201192.928185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941d800. 00000080:00000010:2.1:1788201192.936636:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab50a940. 00000080:00000010:2.1:1788201192.936653:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52016e0. 00010000:00000010:2.1:1788201192.942751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217c00. 00010000:00000010:2.1:1788201192.977907:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab6e7ef00. 00010000:00000010:2.1:1788201192.977921:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941da00. 00000080:00000010:2.1:1788201192.987145:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab780000. 00000080:00000010:2.1:1788201192.987163:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894688e0. 00010000:00000010:2.1:1788201192.987175:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217c00. 00010000:00000010:2.1:1788201192.994651:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab6e7e700. 00010000:00000010:2.1:1788201192.994667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30200. 00000080:00000010:2.1:1788201193.004589:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f6a940. 00000080:00000010:2.1:1788201193.004611:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c60. 00010000:00000010:2.1:1788201193.016838:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6541600. 00010000:00000010:2.1:1788201193.183537:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2c00. 00010000:00000010:2.1:1788201193.183561:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30400. 00010000:00000010:2.1:1788201193.291342:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4d00. 00010000:00000010:2.1:1788201193.291358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30800. 00010000:00000010:2.1:1788201194.915188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6700. 00010000:00000010:2.1:1788201194.915200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32000. 00010000:00000010:2.1:1788201195.374434:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b900. 00010000:00000010:2.1:1788201195.374441:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540200. 00000080:00000010:2.1:1788201195.398025:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c7e300. 00000080:00000010:2.1:1788201195.398222:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52017a0. 00010000:00000010:2.1:1788201195.466632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8200. 00010000:00000010:2.1:1788201195.474514:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b200. 00010000:00000010:2.1:1788201195.474526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6541200. 00000080:00000010:2.1:1788201195.499149:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3711080. 00000080:00000010:2.1:1788201195.499159:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:2.1:1788201195.508167:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217400. 00010000:00000010:2.1:1788201195.762185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b300. 00010000:00000010:2.1:1788201195.762197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87197400. 00000080:00000010:2.1:1788201195.766290:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa935280. 00000080:00000010:2.1:1788201195.766310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018c0. 00010000:00000010:2.1:1788201195.776178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216a00. 00010000:00000010:2.1:1788201195.837131:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217800. 00010000:00000010:2.1:1788201195.846203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b500. 00010000:00000010:2.1:1788201195.846211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87196a00. 00000080:00000010:2.1:1788201195.846218:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5e9080. 00000080:00000010:2.1:1788201195.846223:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468720. 00010000:00000010:2.1:1788201196.044173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3f00. 00010000:00000010:2.1:1788201196.044190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87197e00. 00010000:00000010:2.1:1788201196.321141:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3000. 00010000:00000010:2.1:1788201196.321158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87197600. 00000080:00000010:2.1:1788201196.327493:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5dcd280. 00000080:00000010:2.1:1788201196.327509:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201da0. 00010000:00000010:2.1:1788201196.331162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216600. 00010000:00000010:2.1:1788201196.763533:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3800. 00010000:00000010:2.1:1788201196.763547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9c00. 00010000:00000010:2.1:1788201196.850166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa7532d00. 00010000:00000010:2.1:1788201196.850183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9000. 00010000:00000010:2.1:1788201197.595130:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe108e00. 00010000:00000010:2.1:1788201197.606568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa7532200. 00010000:00000010:2.1:1788201197.606582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11ca00. 00000080:00000010:2.1:1788201197.621668:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa65280. 00000080:00000010:2.1:1788201197.621684:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfa0. 00010000:00000010:2.1:1788201197.640985:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216a00. 00010000:00000010:2.1:1788201197.754185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa7532800. 00010000:00000010:2.1:1788201197.754198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe108600. 00000080:00000010:2.1:1788201197.768142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa544f380. 00000080:00000010:2.1:1788201197.768161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b60. 00010000:00000010:2.1:1788201197.781516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216c00. 00010000:00000010:2.1:1788201197.802170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa7532500. 00010000:00000010:2.1:1788201197.802182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe109e00. 00010000:00000010:2.1:1788201197.914283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72600. 00010000:00000010:2.1:1788201197.914293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe108800. 00000080:00000010:2.1:1788201197.926257:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbca4a40. 00000080:00000010:2.1:1788201197.926274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00010000:00000010:2.1:1788201198.241116:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72000. 00010000:00000010:2.1:1788201198.241124:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045800. 00000080:00000010:2.1:1788201198.246255:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81643180. 00000080:00000010:2.1:1788201198.246436:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2a0. 00010000:00000010:2.1:1788201198.271042:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217a00. 00010000:00000010:2.1:1788201198.484171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72f00. 00010000:00000010:2.1:1788201198.484186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32e00. 00010000:00000010:2.1:1788201199.334183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4b00. 00010000:00000010:2.1:1788201199.334197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15000. 00010000:00000010:2.1:1788201199.383169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4800. 00010000:00000010:2.1:1788201199.383184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15400. 00000080:00000010:2.1:1788201199.391264:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e78000. 00000080:00000010:2.1:1788201199.391277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c220. 00010000:00000010:2.1:1788201199.397878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216c00. 00010000:00000010:2.1:1788201200.126514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9400. 00010000:00000010:2.1:1788201200.141247:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c600. 00010000:00000010:2.1:1788201200.141261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217200. 00000080:00000010:2.1:1788201200.164285:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8e7380. 00000080:00000010:2.1:1788201200.164297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:2.1:1788201200.485225:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cd00. 00010000:00000010:2.1:1788201200.485233:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217c00. 00010000:00000010:2.1:1788201200.824295:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6900. 00010000:00000010:2.1:1788201200.824311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30800. 00000080:00000010:2.1:1788201200.844470:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3452100. 00000080:00000010:2.1:1788201200.844487:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c40. 00010000:00000010:2.1:1788201200.856840:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6600. 00010000:00000010:2.1:1788201201.189590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4100. 00010000:00000010:2.1:1788201201.189605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30200. 00010000:00000010:2.1:1788201201.581183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4800. 00010000:00000010:2.1:1788201201.581190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7000. 00010000:00000010:2.1:1788201201.691834:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4f00. 00010000:00000010:2.1:1788201201.691850:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9c00. 00000080:00000010:2.1:1788201201.701134:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ddd280. 00000080:00000010:2.1:1788201201.701146:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2a0. 00010000:00000010:2.1:1788201201.713143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7000. 00010000:00000010:2.1:1788201201.741455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4900. 00010000:00000010:2.1:1788201201.741469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9200. 00010000:00000010:2.1:1788201202.018039:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2000. 00010000:00000010:2.1:1788201202.018056:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8e00. 00010000:00000010:2.1:1788201202.347286:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b300. 00010000:00000010:2.1:1788201202.347299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417a00. 00010000:00000010:2.1:1788201202.914510:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab6e7e500. 00010000:00000010:2.1:1788201202.914524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577a00. 00010000:00000010:2.1:1788201203.602451:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c500. 00010000:00000010:2.1:1788201203.602467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07e00. 00000080:00000010:2.1:1788201203.631495:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa356eb40. 00000080:00000010:2.1:1788201203.631581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201540. 00010000:00000010:2.1:1788201203.652825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6600. 00010000:00000010:2.1:1788201204.621168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa32a4e00. 00010000:00000010:2.1:1788201204.621186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14800. 00000080:00000010:2.1:1788201204.643727:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfed280. 00000080:00000010:2.1:1788201204.643744:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015e0. 00010000:00000010:2.1:1788201204.651351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00000080:00000010:2.1:1788201204.679885:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35d4200. 00000080:00000010:2.1:1788201204.679901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894685a0. 00010000:00000010:2.1:1788201205.165531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00010000:00000010:2.1:1788201205.177182:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a91477f00. 00010000:00000010:2.1:1788201205.177206:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038fc00. 00000080:00000010:2.1:1788201205.182156:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3528000. 00000080:00000010:2.1:1788201205.182175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:2.1:1788201205.190228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00010000:00000010:2.1:1788201205.211931:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a91477100. 00010000:00000010:2.1:1788201205.211942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e800. 00000080:00000010:2.1:1788201205.216661:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34ed280. 00000080:00000010:2.1:1788201205.216677:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468060. 00010000:00000010:2.1:1788201205.223125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00010000:00000010:2.1:1788201205.243264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a91477300. 00010000:00000010:2.1:1788201205.243285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e000. 00000080:00000010:2.1:1788201205.251794:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa368c200. 00000080:00000010:2.1:1788201205.251815:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ea0. 00010000:00000010:2.1:1788201205.268658:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00010000:00000010:2.1:1788201205.307489:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a91477c00. 00010000:00000010:2.1:1788201205.307496:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038ea00. 00010000:00000010:2.1:1788201205.629163:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51100. 00010000:00000010:2.1:1788201205.629170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031a00. 00010000:00000010:2.1:1788201205.664317:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51b00. 00010000:00000010:2.1:1788201205.664325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:2.1:1788201205.965338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2300. 00010000:00000010:2.1:1788201205.965344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe109200. 00010000:00000010:2.1:1788201206.090162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2d00. 00010000:00000010:2.1:1788201206.090169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe108a00. 00010000:00000010:2.1:1788201206.225338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72100. 00010000:00000010:2.1:1788201206.225346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe109e00. 00000080:00000010:2.1:1788201206.248202:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab735280. 00000080:00000010:2.1:1788201206.248212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2e0. 00010000:00000010:2.1:1788201206.599076:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba400. 00010000:00000010:2.1:1788201206.686191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51000. 00010000:00000010:2.1:1788201206.686207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9e00. 00010000:00000010:2.1:1788201207.011874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6600. 00010000:00000010:2.1:1788201207.040258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2200. 00010000:00000010:2.1:1788201207.040283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8000. 00000080:00000010:2.1:1788201207.117997:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57a6b40. 00000080:00000010:2.1:1788201207.118016:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201840. 00010000:00000010:2.1:1788201207.118026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6600. 00010000:00000010:2.1:1788201207.419424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa7532000. 00010000:00000010:2.1:1788201207.419431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417800. 00010000:00000010:2.1:1788201207.896210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bf00. 00010000:00000010:2.1:1788201207.896223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417a00. 00010000:00000010:2.1:1788201208.288576:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b500. 00010000:00000010:2.1:1788201208.288601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6e00. 00000080:00000010:2.1:1788201208.295300:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4edac0. 00000080:00000010:2.1:1788201208.295313:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c4c0. 00010000:00000010:2.1:1788201208.301262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7600. 00010000:00000010:2.1:1788201208.321715:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2000. 00010000:00000010:2.1:1788201208.321724:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7c00. 00000080:00000010:2.1:1788201208.326323:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5dc0840. 00000080:00000010:2.1:1788201208.326335:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894680c0. 00010000:00000010:2.1:1788201208.337939:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7600. 00010000:00000010:2.1:1788201208.675927:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b100. 00010000:00000010:2.1:1788201208.675942:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102000. 00000080:00000010:2.1:1788201208.701368:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3436300. 00000080:00000010:2.1:1788201208.701391:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc60. 00010000:00000010:2.1:1788201208.900381:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecb00. 00010000:00000010:2.1:1788201208.900388:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102600. 00010000:00000010:2.1:1788201208.924480:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:2.1:1788201208.928811:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecd00. 00010000:00000010:2.1:1788201208.928825:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2c2d000. 00000080:00000010:2.1:1788201208.935423:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57fca40. 00000080:00000010:2.1:1788201208.935452:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172100. 00010000:00000010:2.1:1788201208.947240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00010000:00000010:2.1:1788201209.140445:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15200. 00000080:00000010:2.1:1788201209.159269:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3540840. 00000080:00000010:2.1:1788201209.159442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:2.1:1788201209.383441:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737de00. 00010000:00000010:2.1:1788201209.383448:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2c2d800. 00010000:00000010:2.1:1788201209.660577:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7000. 00010000:00000010:2.1:1788201209.677572:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:2.1:1788201209.677579:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24e00. 00000080:00000010:2.1:1788201209.691263:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85884a40. 00000080:00000010:2.1:1788201209.691284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:2.1:1788201209.691294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7000. 00010000:00000010:2.1:1788201209.705186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952300. 00010000:00000010:2.1:1788201209.705199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24a00. 00000080:00000010:2.1:1788201209.730144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b843c8840. 00000080:00000010:2.1:1788201209.730158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ec0. 00010000:00000010:2.1:1788201210.627475:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952900. 00010000:00000010:2.1:1788201210.627482:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9800. 00000080:00000010:2.1:1788201210.640182:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36a98c0. 00000080:00000010:2.1:1788201210.640200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201fe0. 00010000:00000010:2.1:1788201210.984655:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44200. 00010000:00000010:2.1:1788201210.984669:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b63000. 00000080:00000010:2.1:1788201210.992139:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa40000. 00010000:00000010:2.1:1788201211.000639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0800. 00010000:00000010:2.1:1788201211.401762:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0800. 00010000:00000010:2.1:1788201211.531438:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44e00. 00010000:00000010:2.1:1788201211.531446:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653a600. 00000080:00000010:2.1:1788201211.538144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc96b40. 00000080:00000010:2.1:1788201211.538164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:2.1:1788201211.547143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33e00. 00010000:00000010:2.1:1788201211.560192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909ee00. 00010000:00000010:2.1:1788201211.560209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653a000. 00000080:00000010:2.1:1788201211.569140:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37b18c0. 00000080:00000010:2.1:1788201211.569163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468a40. 00010000:00000010:2.1:1788201211.578528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33e00. 00010000:00000010:2.1:1788201212.362126:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51c00. 00010000:00000010:2.1:1788201212.362133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540600. 00010000:00000010:2.1:1788201212.401164:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51500. 00010000:00000010:2.1:1788201212.401179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540400. 00010000:00000010:2.1:1788201212.650188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b500. 00010000:00000010:2.1:1788201212.650202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00010000:00000010:2.1:1788201212.822219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9bd00. 00010000:00000010:2.1:1788201212.822234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00000080:00000010:2.1:1788201212.835178:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe98000. 00000080:00000010:2.1:1788201212.835199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf20. 00010000:00000010:2.1:1788201212.854041:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3400. 00010000:00000010:2.1:1788201212.969886:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9ba00. 00010000:00000010:2.1:1788201212.969907:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7e00. 00000080:00000010:2.1:1788201213.774164:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab529080. 00000080:00000010:2.1:1788201213.774183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019a0. 00010000:00000010:2.1:1788201213.955183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2600. 00010000:00000010:2.1:1788201213.955195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2c2c000. 00000080:00000010:2.1:1788201213.971751:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56a6b40. 00000080:00000010:2.1:1788201213.971765:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201640. 00010000:00000010:2.1:1788201213.978747:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3800. 00010000:00000010:2.1:1788201213.987319:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2200. 00010000:00000010:2.1:1788201213.987333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2c2ca00. 00000080:00000010:2.1:1788201213.999473:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81496300. 00000080:00000010:2.1:1788201213.999484:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468140. 00010000:00000010:2.1:1788201214.004280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3800. 00010000:00000010:2.1:1788201214.128255:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2d00. 00010000:00000010:2.1:1788201214.128260:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416400. 00000080:00000010:2.1:1788201214.134136:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc6eb40. 00000080:00000010:2.1:1788201214.134153:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd40. 00010000:00000010:2.1:1788201214.139127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3c00. 00010000:00000010:2.1:1788201214.444491:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3e00. 00010000:00000010:2.1:1788201214.510638:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab6e7e200. 00010000:00000010:2.1:1788201214.510651:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038f200. 00000080:00000010:2.1:1788201214.519146:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbeff380. 00000080:00000010:2.1:1788201214.519163:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cac0. 00010000:00000010:2.1:1788201214.519170:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3e00. 00010000:00000010:2.1:1788201214.736491:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab6e7e100. 00010000:00000010:2.1:1788201214.736508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e000. 00000080:00000010:2.1:1788201214.752431:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf76b40. 00000080:00000010:2.1:1788201214.752459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7c0. 00010000:00000010:2.1:1788201214.766133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df2400. 00000080:00000010:2.1:1788201215.611260:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37aa940. 00000080:00000010:2.1:1788201215.611274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:2.1:1788201215.790219:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952b00. 00010000:00000010:2.1:1788201215.790396:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33e00. 00010000:00000010:2.1:1788201215.830135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1c00. 00010000:00000010:2.1:1788201216.230562:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952200. 00010000:00000010:2.1:1788201216.230574:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62a00. 00010000:00000010:2.1:1788201216.283256:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952300. 00010000:00000010:2.1:1788201216.283272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62200. 00000080:00000010:2.1:1788201216.643264:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab528840. 00000080:00000010:2.1:1788201216.643289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:2.1:1788201216.680796:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a91477d00. 00010000:00000010:2.1:1788201216.680803:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11da00. 00010000:00000010:2.1:1788201216.764169:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b600. 00010000:00000010:2.1:1788201216.764176:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11ce00. 00000080:00000010:2.1:1788201216.770746:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56a2940. 00000080:00000010:2.1:1788201216.770770:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cae0. 00010000:00000010:2.1:1788201216.790550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00010000:00000010:2.1:1788201217.038297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a91477800. 00010000:00000010:2.1:1788201217.038311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad11cc00. 00000080:00000010:2.1:1788201217.049145:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa34a40. 00000080:00000010:2.1:1788201217.049164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00000080:00000010:2.1:1788201218.444207:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9ceb40. 00000080:00000010:2.1:1788201218.444224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201d80. 00010000:00000010:2.1:1788201218.684595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84a00. 00010000:00000010:2.1:1788201218.684603:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b400. 00010000:00000010:2.1:1788201218.723198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84e00. 00010000:00000010:2.1:1788201218.723210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a600. 00010000:00000010:2.1:1788201218.800136:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84000. 00010000:00000010:2.1:1788201218.800147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b200. 00010000:00000010:2.1:1788201219.152954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201219.317144:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039900. 00010000:00000010:2.1:1788201219.317175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3c00. 00010000:00000010:2.1:1788201219.455845:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039200. 00010000:00000010:2.1:1788201219.455852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3200. 00010000:00000010:2.1:1788201219.540170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0e00. 00010000:00000010:2.1:1788201219.601250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201219.955164:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039c00. 00010000:00000010:2.1:1788201219.955180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653b200. 00010000:00000010:2.1:1788201220.192530:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039e00. 00010000:00000010:2.1:1788201220.192540:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653b800. 00000080:00000010:2.1:1788201220.201207:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81494200. 00000080:00000010:2.1:1788201220.201219:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cec0. 00010000:00000010:2.1:1788201220.214597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00010000:00000010:2.1:1788201220.233172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6638300. 00010000:00000010:2.1:1788201220.233187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653aa00. 00000080:00000010:2.1:1788201220.246142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbffa940. 00000080:00000010:2.1:1788201220.246161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894685a0. 00010000:00000010:2.1:1788201220.253835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00010000:00000010:2.1:1788201220.603334:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72200. 00010000:00000010:2.1:1788201220.603350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30200. 00010000:00000010:2.1:1788201221.643145:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86500. 00010000:00000010:2.1:1788201221.643158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:2.1:1788201221.707002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7c00. 00010000:00000010:2.1:1788201222.145271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51b00. 00010000:00000010:2.1:1788201222.145287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7600. 00010000:00000010:2.1:1788201222.461198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51500. 00010000:00000010:2.1:1788201222.461214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4a00. 00010000:00000010:2.1:1788201222.525649:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51c00. 00010000:00000010:2.1:1788201222.525662:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4800. 00010000:00000010:2.1:1788201222.633299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51000. 00010000:00000010:2.1:1788201222.633308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4600. 00000080:00000010:2.1:1788201222.638241:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3495280. 00000080:00000010:2.1:1788201222.638256:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c760. 00010000:00000010:2.1:1788201222.649147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb200. 00010000:00000010:2.1:1788201222.672691:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2b00. 00010000:00000010:2.1:1788201222.672701:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5e00. 00010000:00000010:2.1:1788201222.686359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1000. 00010000:00000010:2.1:1788201223.275404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2f00. 00010000:00000010:2.1:1788201223.275420:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00000080:00000010:2.1:1788201223.283476:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe939c0. 00000080:00000010:2.1:1788201223.283502:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:2.1:1788201223.318624:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb200. 00000080:00000010:2.1:1788201223.352224:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8176c200. 00000080:00000010:2.1:1788201223.352243:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ea0. 00010000:00000010:2.1:1788201223.854472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9bd00. 00010000:00000010:2.1:1788201223.854486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3e00. 00010000:00000010:2.1:1788201223.883732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb200. 00010000:00000010:2.1:1788201223.905738:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bc00. 00010000:00000010:2.1:1788201223.905748:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00000080:00000010:2.1:1788201223.905756:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcd18c0. 00000080:00000010:2.1:1788201223.905766:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894685a0. 00010000:00000010:2.1:1788201223.916252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb200. 00000080:00000010:2.1:1788201224.319451:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe00000. 00000080:00000010:2.1:1788201224.319464:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f00. 00010000:00000010:2.1:1788201224.884967:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:2.1:1788201226.123292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeaf100. 00010000:00000010:2.1:1788201226.123307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25e00. 00000080:00000010:2.1:1788201226.818159:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84a00840. 00000080:00000010:2.1:1788201226.818181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce20. 00010000:00000010:2.1:1788201226.858184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6638300. 00010000:00000010:2.1:1788201226.858199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417200. 00010000:00000010:2.1:1788201227.679429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84800. 00010000:00000010:2.1:1788201227.679441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941ce00. 00000080:00000010:2.1:1788201227.684456:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6298c0. 00000080:00000010:2.1:1788201227.684474:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981725e0. 00010000:00000010:2.1:1788201227.689038:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182c00. 00010000:00000010:2.1:1788201228.071730:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84200. 00010000:00000010:2.1:1788201228.071743:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00000080:00000010:2.1:1788201228.077210:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8b5280. 00000080:00000010:2.1:1788201228.077222:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c220. 00010000:00000010:2.1:1788201228.084145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182c00. 00010000:00000010:2.1:1788201228.103738:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84b00. 00010000:00000010:2.1:1788201228.103747:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2800. 00000080:00000010:2.1:1788201228.109906:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe1dac0. 00000080:00000010:2.1:1788201228.109931:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468740. 00010000:00000010:2.1:1788201228.120194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182c00. 00010000:00000010:2.1:1788201228.147718:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84500. 00010000:00000010:2.1:1788201228.147735:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2a00. 00000080:00000010:2.1:1788201228.158201:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81628840. 00000080:00000010:2.1:1788201228.158217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468fe0. 00010000:00000010:2.1:1788201228.158226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182c00. 00010000:00000010:2.1:1788201228.194171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84900. 00010000:00000010:2.1:1788201228.194178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00000080:00000010:2.1:1788201229.413145:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84be2100. 00000080:00000010:2.1:1788201229.413169:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cac0. 00010000:00000010:2.1:1788201229.447479:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bf00. 00010000:00000010:2.1:1788201229.447487:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103800. 00000080:00000010:2.1:1788201229.463199:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab764200. 00000080:00000010:2.1:1788201229.463219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468060. 00010000:00000010:2.1:1788201229.864404:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6dc00. 00010000:00000010:2.1:1788201229.864421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabbc00. 00010000:00000010:2.1:1788201230.062401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:2.1:1788201230.081179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d000. 00010000:00000010:2.1:1788201230.081193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabac00. 00000080:00000010:2.1:1788201230.091544:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3564a40. 00000080:00000010:2.1:1788201230.091554:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201880. 00010000:00000010:2.1:1788201230.100189:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82000. 00010000:00000010:2.1:1788201230.115294:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d400. 00010000:00000010:2.1:1788201230.115305:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00000080:00000010:2.1:1788201230.126981:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35318c0. 00000080:00000010:2.1:1788201230.127000:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ca0. 00010000:00000010:2.1:1788201230.566227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86700. 00010000:00000010:2.1:1788201230.566247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8e00. 00000080:00000010:2.1:1788201230.576287:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6bb9c0. 00000080:00000010:2.1:1788201230.576307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f20. 00010000:00000010:2.1:1788201230.584142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:2.1:1788201231.174282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6400. 00010000:00000010:2.1:1788201231.174288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577400. 00010000:00000010:2.1:1788201231.441372:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeafb00. 00010000:00000010:2.1:1788201231.441388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045600. 00000080:00000010:2.1:1788201231.464233:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa912100. 00000080:00000010:2.1:1788201231.464245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52017c0. 00010000:00000010:2.1:1788201231.467594:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83200. 00010000:00000010:2.1:1788201231.680281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeaf700. 00010000:00000010:2.1:1788201231.680299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217a00. 00010000:00000010:2.1:1788201231.699229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1c00. 00010000:00000010:2.1:1788201231.717191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeafc00. 00010000:00000010:2.1:1788201231.717205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217200. 00000080:00000010:2.1:1788201231.721326:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36498c0. 00000080:00000010:2.1:1788201231.721345:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981721a0. 00010000:00000010:2.1:1788201231.739136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83200. 00010000:00000010:2.1:1788201232.465358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3f00. 00010000:00000010:2.1:1788201232.465373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183c00. 00000080:00000010:2.1:1788201232.471432:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3493180. 00000080:00000010:2.1:1788201232.471448:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019a0. 00010000:00000010:2.1:1788201232.484756:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82a00. 00010000:00000010:2.1:1788201232.500949:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3400. 00010000:00000010:2.1:1788201232.500963:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183800. 00000080:00000010:2.1:1788201232.514378:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85955280. 00000080:00000010:2.1:1788201232.514395:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468560. 00010000:00000010:2.1:1788201232.531322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82a00. 00010000:00000010:2.1:1788201232.970460:0:256135:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00010000:00000010:2.1:1788201233.110822:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7700. 00010000:00000010:2.1:1788201233.110828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183e00. 00000080:00000010:2.1:1788201233.118085:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab56ca40. 00000080:00000010:2.1:1788201233.118113:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c940. 00010000:00000010:2.1:1788201233.128145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabaa00. 00010000:00000010:2.1:1788201233.134413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7500. 00010000:00000010:2.1:1788201233.134425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182e00. 00000080:00000010:2.1:1788201233.141254:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34db9c0. 00000080:00000010:2.1:1788201233.141272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468740. 00010000:00000010:2.1:1788201233.155695:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabaa00. 00010000:00000010:2.1:1788201233.283315:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeaf400. 00010000:00000010:2.1:1788201233.283329:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2400. 00000080:00000010:2.1:1788201233.293384:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7d2100. 00000080:00000010:2.1:1788201233.293401:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8a0. 00010000:00000010:2.1:1788201233.302770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabae00. 00000080:00000010:2.1:1788201233.368140:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3726300. 00000080:00000010:2.1:1788201233.368164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:2.1:1788201233.388148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb600. 00010000:00000010:2.1:1788201233.623403:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c100. 00010000:00000010:2.1:1788201233.623410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe109400. 00000080:00000010:2.1:1788201233.627553:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5478840. 00000080:00000010:2.1:1788201233.627569:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a20. 00010000:00000010:2.1:1788201233.639146:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb800. 00010000:00000010:2.1:1788201233.797262:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84300. 00010000:00000010:2.1:1788201233.797418:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe108600. 00010000:00000010:2.1:1788201234.011594:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22100. 00010000:00000010:2.1:1788201234.011609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbe109000. 00000080:00000010:2.1:1788201234.034246:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ded280. 00000080:00000010:2.1:1788201234.034264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014a0. 00010000:00000010:2.1:1788201234.175557:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:2.1:1788201234.473228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2a00. 00010000:00000010:2.1:1788201234.473238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9e02e400. 00010000:00000010:2.1:1788201234.575191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00010000:00000010:2.1:1788201234.601260:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2200. 00010000:00000010:2.1:1788201234.601267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9e02e600. 00010000:00000010:2.1:1788201235.327329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb500. 00010000:00000010:2.1:1788201235.327341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82a00. 00000080:00000010:2.1:1788201235.339996:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbed0000. 00000080:00000010:2.1:1788201235.340018:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ccc0. 00010000:00000010:2.1:1788201235.350202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540a00. 00010000:00000010:2.1:1788201236.664916:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a82eb2b00. 00010000:00000010:2.1:1788201236.664933:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8e00. 00010000:00000010:2.1:1788201236.977419:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124ad00. 00010000:00000010:2.1:1788201236.977430:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102600. 00000080:00000010:2.1:1788201236.983496:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35fdac0. 00000080:00000010:2.1:1788201236.983515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201900. 00010000:00000010:2.1:1788201237.128038:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f300. 00010000:00000010:2.1:1788201237.128050:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102800. 00010000:00000010:2.1:1788201237.204226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f100. 00010000:00000010:2.1:1788201237.204240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103a00. 00010000:00000010:2.1:1788201237.380210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6541a00. 00010000:00000010:2.1:1788201237.682930:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeafb00. 00010000:00000010:2.1:1788201237.682950:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4400. 00000080:00000010:2.1:1788201237.691365:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf0f380. 00000080:00000010:2.1:1788201237.691383:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c6e0. 00010000:00000010:2.1:1788201237.718590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540600. 00010000:00000010:2.1:1788201237.729329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeaf200. 00010000:00000010:2.1:1788201237.729344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5400. 00010000:00000010:2.1:1788201238.660594:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6300. 00010000:00000010:2.1:1788201238.660637:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabaa00. 00010000:00000010:2.1:1788201238.841367:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6e00. 00010000:00000010:2.1:1788201238.841385:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb200. 00000080:00000010:2.1:1788201238.841394:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54ba940. 00000080:00000010:2.1:1788201238.841410:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cae0. 00010000:00000010:2.1:1788201238.852259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217c00. 00000080:00000010:2.1:1788201239.186211:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fddac0. 00000080:00000010:2.1:1788201239.186221:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c140. 00010000:00000010:2.1:1788201239.197566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217400. 00010000:00000010:2.1:1788201239.220724:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24b00. 00010000:00000010:2.1:1788201239.220742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9400. 00000080:00000010:2.1:1788201239.235605:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35ed280. 00000080:00000010:2.1:1788201239.235617:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468f00. 00010000:00000010:2.1:1788201239.239371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217400. 00000080:00000010:2.1:1788201239.272423:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34b8840. 00000080:00000010:2.1:1788201239.272441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894681e0. 00010000:00000010:2.1:1788201239.277633:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216600. 00010000:00000010:2.1:1788201239.290527:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeafe00. 00010000:00000010:2.1:1788201239.290538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9e00. 00000080:00000010:2.1:1788201239.297180:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf09080. 00000080:00000010:2.1:1788201239.297195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468140. 00010000:00000010:2.1:1788201239.301422:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216600. 00010000:00000010:2.1:1788201239.317325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaaeaf400. 00010000:00000010:2.1:1788201239.317344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8800. 00000080:00000010:2.1:1788201239.327388:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37d0840. 00000080:00000010:2.1:1788201239.327409:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468860. 00010000:00000010:2.1:1788201239.340193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216600. 00010000:00000010:2.1:1788201239.479268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959e00. 00010000:00000010:2.1:1788201239.479282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25e00. 00000080:00000010:2.1:1788201239.487254:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3478840. 00000080:00000010:2.1:1788201239.487281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52016e0. 00010000:00000010:2.1:1788201239.505773:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216a00. 00010000:00000010:2.1:1788201239.948263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b400. 00010000:00000010:2.1:1788201239.948277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33c00. 00000080:00000010:2.1:1788201239.956256:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5451080. 00000080:00000010:2.1:1788201239.956286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201bc0. 00000080:00000010:2.1:1788201240.051220:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57be300. 00000080:00000010:2.1:1788201240.051236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894680c0. 00010000:00000010:2.1:1788201240.070641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216a00. 00010000:00000010:2.1:1788201240.254297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9bd00. 00010000:00000010:2.1:1788201240.254312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32a00. 00000080:00000010:2.1:1788201240.273331:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab78d280. 00000080:00000010:2.1:1788201240.273350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f00. 00010000:00000010:2.1:1788201240.290572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217400. 00010000:00000010:2.1:1788201240.337179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b600. 00010000:00000010:2.1:1788201240.337195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33e00. 00000080:00000010:2.1:1788201240.349251:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817af380. 00000080:00000010:2.1:1788201240.349267:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468fe0. 00010000:00000010:2.1:1788201240.366252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217400. 00010000:00000010:2.1:1788201240.550115:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b400. 00010000:00000010:2.1:1788201240.550126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870000. 00010000:00000010:2.1:1788201240.828198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b200. 00010000:00000010:2.1:1788201240.828212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00000080:00000010:2.1:1788201241.223170:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c92940. 00000080:00000010:2.1:1788201241.223183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:2.1:1788201241.474191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bc00. 00010000:00000010:2.1:1788201241.474212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6541200. 00000080:00000010:2.1:1788201241.478149:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa540a940. 00000080:00000010:2.1:1788201241.478168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c400. 00010000:00000010:2.1:1788201241.494153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216800. 00010000:00000010:2.1:1788201242.486190:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6300. 00010000:00000010:2.1:1788201242.486203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabac00. 00010000:00000010:2.1:1788201242.616652:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6f00. 00010000:00000010:2.1:1788201242.616666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5800. 00010000:00000010:2.1:1788201242.688153:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6700. 00010000:00000010:2.1:1788201242.688174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5a00. 00010000:00000010:2.1:1788201242.825313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f000. 00010000:00000010:2.1:1788201242.825320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5400. 00000080:00000010:2.1:1788201242.829142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab51b180. 00000080:00000010:2.1:1788201242.829157:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c5a0. 00000080:00000010:2.1:1788201243.037433:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7818c0. 00000080:00000010:2.1:1788201243.037445:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019a0. 00010000:00000010:2.1:1788201243.781886:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96ff00. 00010000:00000010:2.1:1788201243.781895:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07600. 00000080:00000010:2.1:1788201243.781904:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81494a40. 00000080:00000010:2.1:1788201243.781914:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce20. 00010000:00000010:2.1:1788201243.798218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7c00. 00010000:00000010:2.1:1788201243.804379:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96fa00. 00010000:00000010:2.1:1788201243.804392:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06800. 00000080:00000010:2.1:1788201243.814286:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7c6b40. 00000080:00000010:2.1:1788201243.814302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468100. 00010000:00000010:2.1:1788201244.303308:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84900. 00010000:00000010:2.1:1788201244.303321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:2.1:1788201244.369174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4200. 00010000:00000010:2.1:1788201244.369188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201244.382326:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54f7380. 00000080:00000010:2.1:1788201244.382350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2c0. 00010000:00000010:2.1:1788201244.599722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00010000:00000010:2.1:1788201244.623510:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863ca00. 00010000:00000010:2.1:1788201244.623708:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216c00. 00000080:00000010:2.1:1788201244.623716:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a816498c0. 00000080:00000010:2.1:1788201244.623724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce80. 00010000:00000010:2.1:1788201244.627430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7000. 00010000:00000010:2.1:1788201244.812174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c000. 00010000:00000010:2.1:1788201244.812191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00000080:00000010:2.1:1788201244.819701:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbffb9c0. 00000080:00000010:2.1:1788201244.819724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:2.1:1788201244.835137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6e00. 00010000:00000010:2.1:1788201244.868179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c100. 00010000:00000010:2.1:1788201244.868196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201245.596190:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bb00. 00010000:00000010:2.1:1788201245.596199:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045800. 00000080:00000010:2.1:1788201245.610185:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8a98c0. 00000080:00000010:2.1:1788201245.610195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c40. 00010000:00000010:2.1:1788201245.615151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7800. 00010000:00000010:2.1:1788201245.632779:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7a00. 00010000:00000010:2.1:1788201245.632793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6400. 00000080:00000010:2.1:1788201246.136147:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84b73180. 00000080:00000010:2.1:1788201246.136161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:2.1:1788201247.203403:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959100. 00010000:00000010:2.1:1788201247.203415:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83a00. 00000080:00000010:2.1:1788201247.211158:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c82100. 00000080:00000010:2.1:1788201247.211178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c6c0. 00010000:00000010:2.1:1788201247.237386:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eee00. 00010000:00000010:2.1:1788201247.408209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84200. 00010000:00000010:2.1:1788201247.408217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24e00. 00010000:00000010:2.1:1788201247.699167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:2.1:1788201247.699174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870400. 00010000:00000010:2.1:1788201247.807922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c100. 00010000:00000010:2.1:1788201247.807939:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870600. 00000080:00000010:2.1:1788201247.820135:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8590b180. 00000080:00000010:2.1:1788201247.820146:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c760. 00010000:00000010:2.1:1788201247.828743:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00010000:00000010:2.1:1788201248.068387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:2.1:1788201248.068402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216600. 00010000:00000010:2.1:1788201248.144213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7e00. 00000080:00000010:2.1:1788201248.345233:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57ec200. 00000080:00000010:2.1:1788201248.345244:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015e0. 00010000:00000010:2.1:1788201248.752154:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bb00. 00010000:00000010:2.1:1788201248.752172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:2.1:1788201249.744155:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7300. 00010000:00000010:2.1:1788201249.744162:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6400. 00010000:00000010:2.1:1788201250.126182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7800. 00010000:00000010:2.1:1788201250.126195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7600. 00010000:00000010:2.1:1788201250.832262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72c00. 00010000:00000010:2.1:1788201250.832285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416a00. 00010000:00000010:2.1:1788201251.464291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b900. 00010000:00000010:2.1:1788201251.464306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00000080:00000010:2.1:1788201251.477179:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815d6300. 00000080:00000010:2.1:1788201251.477204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019c0. 00010000:00000010:2.1:1788201251.483246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eec00. 00010000:00000010:2.1:1788201251.765606:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952400. 00010000:00000010:2.1:1788201251.765618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3e00. 00000080:00000010:2.1:1788201251.777418:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815f18c0. 00000080:00000010:2.1:1788201251.777434:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:2.1:1788201252.489473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44b00. 00010000:00000010:2.1:1788201252.489490:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102000. 00000080:00000010:2.1:1788201252.523599:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe08840. 00000080:00000010:2.1:1788201252.523615:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201900. 00010000:00000010:2.1:1788201253.490222:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b300. 00010000:00000010:2.1:1788201253.490232:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30400. 00000080:00000010:2.1:1788201253.505588:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e1ca40. 00000080:00000010:2.1:1788201253.505613:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52017a0. 00010000:00000010:2.1:1788201253.517639:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00010000:00000010:2.1:1788201253.766128:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82800. 00010000:00000010:2.1:1788201253.787174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b600. 00010000:00000010:2.1:1788201253.787188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30800. 00000080:00000010:2.1:1788201253.795137:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa568eb40. 00000080:00000010:2.1:1788201253.795151:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018c0. 00010000:00000010:2.1:1788201253.808138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00010000:00000010:2.1:1788201253.831459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bb00. 00010000:00000010:2.1:1788201253.831474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df2600. 00000080:00000010:2.1:1788201253.841718:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c37380. 00000080:00000010:2.1:1788201253.855719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ca0. 00000080:00000010:2.1:1788201254.281129:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84bdb9c0. 00000080:00000010:2.1:1788201254.281147:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c80. 00010000:00000010:2.1:1788201254.293531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eee00. 00010000:00000010:2.1:1788201254.390327:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50100. 00010000:00000010:2.1:1788201254.390357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a600. 00010000:00000010:2.1:1788201254.615240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e012c00. 00010000:00000010:2.1:1788201254.615251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62400. 00000080:00000010:2.1:1788201255.162627:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa561dac0. 00000080:00000010:2.1:1788201255.162646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172320. 00010000:00000010:2.1:1788201255.386448:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f400. 00010000:00000010:2.1:1788201255.386456:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6800. 00000080:00000010:2.1:1788201255.400158:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5444a40. 00000080:00000010:2.1:1788201255.400183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201be0. 00010000:00000010:2.1:1788201255.400193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201255.468249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f000. 00010000:00000010:2.1:1788201255.468263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941d800. 00000080:00000010:2.1:1788201255.475513:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57df380. 00000080:00000010:2.1:1788201255.475533:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc40. 00010000:00000010:2.1:1788201255.487471:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef000. 00010000:00000010:2.1:1788201255.508323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96fc00. 00010000:00000010:2.1:1788201255.508338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941da00. 00000080:00000010:2.1:1788201255.516329:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e12100. 00000080:00000010:2.1:1788201255.516342:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468de0. 00010000:00000010:2.1:1788201255.584370:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f900. 00010000:00000010:2.1:1788201255.584383:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941ce00. 00000080:00000010:2.1:1788201255.589179:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cdd280. 00000080:00000010:2.1:1788201255.589191:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cbc0. 00010000:00000010:2.1:1788201255.596548:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00000080:00000010:2.1:1788201255.623863:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab53dac0. 00000080:00000010:2.1:1788201255.623878:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468760. 00010000:00000010:2.1:1788201256.545550:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72e00. 00010000:00000010:2.1:1788201256.545559:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24a00. 00010000:00000010:2.1:1788201256.584258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72400. 00010000:00000010:2.1:1788201256.584267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24000. 00010000:00000010:2.1:1788201256.613305:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72f00. 00010000:00000010:2.1:1788201256.613325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25a00. 00010000:00000010:2.1:1788201256.750735:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72100. 00010000:00000010:2.1:1788201256.750759:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00000080:00000010:2.1:1788201256.761152:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3542940. 00000080:00000010:2.1:1788201256.761170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c440. 00010000:00000010:2.1:1788201257.020184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b100. 00010000:00000010:2.1:1788201257.020196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83a00. 00000080:00000010:2.1:1788201257.034144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c91080. 00000080:00000010:2.1:1788201257.034156:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce80. 00010000:00000010:2.1:1788201257.185204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201257.893048:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e600. 00010000:00000010:2.1:1788201257.893063:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b200. 00010000:00000010:2.1:1788201258.141325:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7a00. 00010000:00000010:2.1:1788201258.141352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a200. 00000080:00000010:2.1:1788201258.155157:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa549ca40. 00000080:00000010:2.1:1788201258.155174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c9e0. 00010000:00000010:2.1:1788201258.377292:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7800. 00010000:00000010:2.1:1788201258.377304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30800. 00000080:00000010:2.1:1788201258.454142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81503180. 00000080:00000010:2.1:1788201258.454163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:2.1:1788201258.519176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7700. 00010000:00000010:2.1:1788201258.519190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c31e00. 00000080:00000010:2.1:1788201258.540160:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa994a40. 00000080:00000010:2.1:1788201258.540182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ee0. 00010000:00000010:2.1:1788201258.550566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00000080:00000010:2.1:1788201258.676045:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00010000:00000010:2.1:1788201258.711147:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7500. 00010000:00000010:2.1:1788201258.711153:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30200. 00000080:00000010:2.1:1788201258.716672:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34c9080. 00000080:00000010:2.1:1788201258.716918:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468940. 00010000:00000010:2.1:1788201258.731341:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201259.074973:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b300. 00010000:00000010:2.1:1788201259.074981:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653ac00. 00000080:00000010:2.1:1788201259.074989:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc25280. 00000080:00000010:2.1:1788201259.074995:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015e0. 00010000:00000010:2.1:1788201259.097196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201259.446653:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51700. 00010000:00000010:2.1:1788201259.446664:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103a00. 00000080:00000010:2.1:1788201259.453446:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54d5280. 00000080:00000010:2.1:1788201259.453459:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c020. 00010000:00000010:2.1:1788201259.460146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201259.479187:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51f00. 00010000:00000010:2.1:1788201259.479195:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103400. 00000080:00000010:2.1:1788201259.489333:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3490000. 00000080:00000010:2.1:1788201259.489349:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e60. 00010000:00000010:2.1:1788201259.495241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201259.511889:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51800. 00010000:00000010:2.1:1788201259.511896:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00000080:00000010:2.1:1788201259.518430:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf1b9c0. 00000080:00000010:2.1:1788201259.518451:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894685a0. 00010000:00000010:2.1:1788201259.527492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201259.595216:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51d00. 00010000:00000010:2.1:1788201259.595225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00010000:00000010:2.1:1788201260.077129:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24100. 00010000:00000010:2.1:1788201260.077139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8800. 00010000:00000010:2.1:1788201260.343420:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959300. 00010000:00000010:2.1:1788201260.343428:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9200. 00000080:00000010:2.1:1788201260.350527:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84a1a940. 00000080:00000010:2.1:1788201260.350540:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c260. 00010000:00000010:2.1:1788201260.355152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eec00. 00010000:00000010:2.1:1788201260.613484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653ac00. 00010000:00000010:2.1:1788201260.654169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72100. 00010000:00000010:2.1:1788201260.654182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540c00. 00000080:00000010:2.1:1788201261.388424:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814bb180. 00000080:00000010:2.1:1788201261.388449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f20. 00010000:00000010:2.1:1788201261.853242:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72300. 00010000:00000010:2.1:1788201261.853258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabac00. 00000080:00000010:2.1:1788201261.865665:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcdf380. 00000080:00000010:2.1:1788201261.865680:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201880. 00010000:00000010:2.1:1788201261.885180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00010000:00000010:2.1:1788201262.579957:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b300. 00010000:00000010:2.1:1788201262.579965:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e000. 00000080:00000010:2.1:1788201262.584383:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc318c0. 00000080:00000010:2.1:1788201262.584403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201cc0. 00010000:00000010:2.1:1788201262.595374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201262.786227:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c500. 00010000:00000010:2.1:1788201262.786243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038ec00. 00010000:00000010:2.1:1788201263.613765:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44b00. 00010000:00000010:2.1:1788201263.613782:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8200. 00000080:00000010:2.1:1788201263.628402:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa941080. 00000080:00000010:2.1:1788201263.628425:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8a0. 00010000:00000010:2.1:1788201263.644117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00000080:00000010:2.1:1788201263.701004:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c90840. 00000080:00000010:2.1:1788201263.701014:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894680a0. 00010000:00000010:2.1:1788201263.714957:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef000. 00010000:00000010:2.1:1788201263.758365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44f00. 00010000:00000010:2.1:1788201263.758376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576000. 00010000:00000010:2.1:1788201263.903134:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44900. 00010000:00000010:2.1:1788201263.903142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577800. 00010000:00000010:2.1:1788201264.111217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62a00. 00000080:00000010:2.1:1788201264.532018:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfeeb40. 00000080:00000010:2.1:1788201264.532032:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201820. 00010000:00000010:2.1:1788201264.549185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00010000:00000010:2.1:1788201264.571364:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c200. 00010000:00000010:2.1:1788201264.571376:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217800. 00010000:00000010:2.1:1788201264.885578:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959000. 00010000:00000010:2.1:1788201264.885593:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07000. 00010000:00000010:2.1:1788201265.684462:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51300. 00010000:00000010:2.1:1788201265.684469:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217a00. 00000080:00000010:2.1:1788201265.691996:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa351a940. 00000080:00000010:2.1:1788201265.692008:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014e0. 00010000:00000010:2.1:1788201265.696249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00000080:00000010:2.1:1788201265.728447:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fba940. 00000080:00000010:2.1:1788201265.728466:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468560. 00010000:00000010:2.1:1788201265.733389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201265.787117:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4a00. 00010000:00000010:2.1:1788201266.228139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:2.1:1788201266.962501:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c500. 00010000:00000010:2.1:1788201266.962520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8c00. 00000080:00000010:2.1:1788201267.085176:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36e4200. 00000080:00000010:2.1:1788201267.085205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172160. 00010000:00000010:2.1:1788201267.096138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00010000:00000010:2.1:1788201267.147168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c700. 00010000:00000010:2.1:1788201267.147181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103a00. 00010000:00000010:2.1:1788201267.485182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6900. 00010000:00000010:2.1:1788201267.485191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102000. 00010000:00000010:2.1:1788201267.573396:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eec00. 00010000:00000010:2.1:1788201267.601086:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7700. 00010000:00000010:2.1:1788201267.601124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870600. 00010000:00000010:2.1:1788201267.623144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62000. 00000080:00000010:2.1:1788201267.882831:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa58840. 00000080:00000010:2.1:1788201267.882847:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf20. 00010000:00000010:2.1:1788201268.493526:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7c00. 00010000:00000010:2.1:1788201268.493540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1e00. 00010000:00000010:2.1:1788201269.712385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201269.739413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6de00. 00010000:00000010:2.1:1788201269.739432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038ec00. 00000080:00000010:2.1:1788201269.757197:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8fca40. 00000080:00000010:2.1:1788201269.757213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cbc0. 00010000:00000010:2.1:1788201269.762148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201270.021149:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24b00. 00010000:00000010:2.1:1788201270.021166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038ea00. 00010000:00000010:2.1:1788201270.303161:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24700. 00010000:00000010:2.1:1788201270.303176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83be00. 00010000:00000010:2.1:1788201270.591314:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959d00. 00010000:00000010:2.1:1788201270.591331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a000. 00000080:00000010:2.1:1788201270.602375:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6b7380. 00000080:00000010:2.1:1788201270.602392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce20. 00010000:00000010:2.1:1788201270.614437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efa00. 00000080:00000010:2.1:1788201270.668856:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab69b9c0. 00000080:00000010:2.1:1788201270.668873:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468300. 00010000:00000010:2.1:1788201270.682177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efa00. 00000080:00000010:2.1:1788201272.096757:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81581080. 00000080:00000010:2.1:1788201272.096778:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf60. 00010000:00000010:2.1:1788201272.310239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909ee00. 00010000:00000010:2.1:1788201272.310255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90940a00. 00000080:00000010:2.1:1788201272.335889:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6c6b40. 00000080:00000010:2.1:1788201272.335901:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb00. 00010000:00000010:2.1:1788201273.375796:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50300. 00010000:00000010:2.1:1788201273.375808:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62a00. 00000080:00000010:2.1:1788201273.544720:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ede300. 00000080:00000010:2.1:1788201273.549228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:2.1:1788201273.774139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149c000. 00010000:00000010:2.1:1788201274.497409:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e012000. 00010000:00000010:2.1:1788201274.497421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15000. 00000080:00000010:2.1:1788201274.531545:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa932100. 00000080:00000010:2.1:1788201274.531567:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201680. 00010000:00000010:2.1:1788201274.540466:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540c00. 00010000:00000010:2.1:1788201275.464180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952200. 00010000:00000010:2.1:1788201275.464198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7200. 00000080:00000010:2.1:1788201275.489827:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81651080. 00000080:00000010:2.1:1788201275.489856:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c260. 00010000:00000010:2.1:1788201275.506654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540000. 00000080:00000010:2.1:1788201275.573757:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa908840. 00000080:00000010:2.1:1788201275.573775:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894682c0. 00010000:00000010:2.1:1788201278.312306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72000. 00010000:00000010:2.1:1788201278.312315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0800. 00010000:00000010:2.1:1788201278.560280:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72f00. 00010000:00000010:2.1:1788201278.560289:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00000080:00000010:2.1:1788201278.569702:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b843cf380. 00000080:00000010:2.1:1788201278.569720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f40. 00010000:00000010:2.1:1788201278.578215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4600. 00010000:00000010:2.1:1788201278.602171:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72a00. 00010000:00000010:2.1:1788201278.602181:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00000080:00000010:2.1:1788201278.612576:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3513180. 00000080:00000010:2.1:1788201278.612600:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468520. 00010000:00000010:2.1:1788201279.148443:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b200. 00010000:00000010:2.1:1788201279.148457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb200. 00000080:00000010:2.1:1788201279.155294:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab64c200. 00000080:00000010:2.1:1788201279.155308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b00. 00010000:00000010:2.1:1788201279.162141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00010000:00000010:2.1:1788201279.183724:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9bd00. 00010000:00000010:2.1:1788201279.183732:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba600. 00000080:00000010:2.1:1788201279.188134:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81410840. 00000080:00000010:2.1:1788201279.188146:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686c0. 00010000:00000010:2.1:1788201279.194524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00010000:00000010:2.1:1788201280.105503:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b400. 00010000:00000010:2.1:1788201280.105520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30400. 00010000:00000010:2.1:1788201280.205483:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bb00. 00010000:00000010:2.1:1788201280.205495:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30200. 00000080:00000010:2.1:1788201280.217461:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ea8000. 00000080:00000010:2.1:1788201280.217477:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00010000:00000010:2.1:1788201280.225136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5800. 00000080:00000010:2.1:1788201280.712579:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca20. 00010000:00000010:2.1:1788201281.097421:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952b00. 00010000:00000010:2.1:1788201281.097427:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83400. 00010000:00000010:2.1:1788201281.229666:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952400. 00010000:00000010:2.1:1788201281.229681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00000080:00000010:2.1:1788201281.259399:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37b8840. 00000080:00000010:2.1:1788201281.259421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:2.1:1788201281.280138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480e00. 00010000:00000010:2.1:1788201281.524022:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952100. 00010000:00000010:2.1:1788201281.524035:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941d400. 00000080:00000010:2.1:1788201282.193229:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d9ca40. 00000080:00000010:2.1:1788201282.193246:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018c0. 00010000:00000010:2.1:1788201282.206543:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5600. 00000080:00000010:2.1:1788201282.768222:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36f39c0. 00000080:00000010:2.1:1788201282.768234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201480. 00010000:00000010:2.1:1788201282.771215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00010000:00000010:2.1:1788201282.787159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50f00. 00010000:00000010:2.1:1788201282.787172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00000080:00000010:2.1:1788201282.799747:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f04a40. 00000080:00000010:2.1:1788201282.799760:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468f60. 00010000:00000010:2.1:1788201282.805104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00010000:00000010:2.1:1788201282.822291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50100. 00010000:00000010:2.1:1788201282.822325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5000. 00000080:00000010:2.1:1788201282.837033:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf49080. 00000080:00000010:2.1:1788201282.837053:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468cc0. 00010000:00000010:2.1:1788201282.837062:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00010000:00000010:2.1:1788201282.972212:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44100. 00010000:00000010:2.1:1788201282.972220:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4a00. 00010000:00000010:2.1:1788201283.333340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44000. 00010000:00000010:2.1:1788201283.333357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5200. 00000080:00000010:2.1:1788201283.347334:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5aeb40. 00000080:00000010:2.1:1788201283.347356:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7c0. 00000080:00000010:2.1:1788201283.422243:0:1032:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3775ac0. 00000080:00000010:2.1:1788201283.422253:0:1032:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e00. 00010000:00000010:2.1:1788201283.908272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e012900. 00010000:00000010:2.1:1788201283.908290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0400. 00000080:00000010:2.1:1788201283.937452:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f35280. 00000080:00000010:2.1:1788201283.937462:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:2.1:1788201283.942912:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5e00. 00010000:00000010:2.1:1788201284.044164:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e300. 00010000:00000010:2.1:1788201284.044174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1000. 00010000:00000010:2.1:1788201284.483282:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3cd00. 00010000:00000010:2.1:1788201284.483296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6600. 00000080:00000010:2.1:1788201284.500536:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbdbdac0. 00000080:00000010:2.1:1788201284.500555:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201700. 00010000:00000010:2.1:1788201284.509382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00010000:00000010:2.1:1788201285.623794:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c400. 00010000:00000010:2.1:1788201285.623812:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32a00. 00000080:00000010:2.1:1788201285.634213:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab652100. 00000080:00000010:2.1:1788201285.634224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c80. 00010000:00000010:2.1:1788201286.482350:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b600. 00010000:00000010:2.1:1788201286.482366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00000080:00000010:2.1:1788201286.500559:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbff8840. 00000080:00000010:2.1:1788201286.500578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201be0. 00010000:00000010:2.1:1788201286.507269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83be00. 00010000:00000010:2.1:1788201286.977293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50000. 00010000:00000010:2.1:1788201286.977305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2c00. 00010000:00000010:2.1:1788201287.483506:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72c00. 00010000:00000010:2.1:1788201287.483518:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00000080:00000010:2.1:1788201287.492189:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56b39c0. 00000080:00000010:2.1:1788201287.492208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:2.1:1788201287.507152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b400. 00010000:00000010:2.1:1788201287.539165:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72900. 00010000:00000010:2.1:1788201287.539172:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83200. 00000080:00000010:2.1:1788201287.548142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eae300. 00000080:00000010:2.1:1788201287.548161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e40. 00010000:00000010:2.1:1788201287.561435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b400. 00010000:00000010:2.1:1788201287.707618:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44900. 00010000:00000010:2.1:1788201287.707625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83400. 00010000:00000010:2.1:1788201288.050289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50b00. 00010000:00000010:2.1:1788201288.050305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83e00. 00000080:00000010:2.1:1788201288.059136:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56fd280. 00000080:00000010:2.1:1788201288.059152:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201820. 00010000:00000010:2.1:1788201288.066234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83ac00. 00010000:00000010:2.1:1788201288.088155:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50d00. 00010000:00000010:2.1:1788201288.088160:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83c00. 00000080:00000010:2.1:1788201288.093140:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa354b180. 00000080:00000010:2.1:1788201288.093155:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468140. 00010000:00000010:2.1:1788201288.093164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83ac00. 00010000:00000010:2.1:1788201288.127327:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50a00. 00010000:00000010:2.1:1788201288.127344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83800. 00000080:00000010:2.1:1788201288.374180:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5dbe300. 00000080:00000010:2.1:1788201288.374194:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:2.1:1788201288.383371:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b600. 00010000:00000010:2.1:1788201288.792980:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952300. 00010000:00000010:2.1:1788201288.792993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417a00. 00000080:00000010:2.1:1788201288.798508:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c9c200. 00000080:00000010:2.1:1788201288.798519:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201ae0. 00010000:00000010:2.1:1788201288.803440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a600. 00010000:00000010:2.1:1788201288.824183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952200. 00010000:00000010:2.1:1788201288.824573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416400. 00000080:00000010:2.1:1788201288.837270:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b843cb180. 00000080:00000010:2.1:1788201288.837297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686c0. 00010000:00000010:2.1:1788201288.856223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a600. 00000080:00000010:2.1:1788201289.150754:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab773180. 00000080:00000010:2.1:1788201289.150765:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c040. 00010000:00000010:2.1:1788201289.199172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b000. 00010000:00000010:2.1:1788201289.199194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4600. 00000080:00000010:2.1:1788201289.228673:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817f3180. 00000080:00000010:2.1:1788201289.228695:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e00. 00010000:00000010:2.1:1788201289.247148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83ac00. 00000080:00000010:2.1:1788201289.589327:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab547380. 00000080:00000010:2.1:1788201289.589342:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201640. 00000080:00000010:2.1:1788201289.601064:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbd81080. 00000080:00000010:2.1:1788201289.601075:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c60. 00010000:00000010:2.1:1788201289.654171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88d9b600. 00010000:00000010:2.1:1788201289.654197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5e00. 00000080:00000010:2.1:1788201289.664558:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84066300. 00000080:00000010:2.1:1788201289.664578:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468640. 00010000:00000010:2.1:1788201289.680333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a000. 00010000:00000010:2.1:1788201290.053311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7c00. 00010000:00000010:2.1:1788201290.053326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00000080:00000010:2.1:1788201290.067293:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84836b40. 00000080:00000010:2.1:1788201290.067305:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201880. 00010000:00000010:2.1:1788201290.073145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a400. 00010000:00000010:2.1:1788201290.099969:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7d00. 00010000:00000010:2.1:1788201290.099981:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4200. 00000080:00000010:2.1:1788201290.129816:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3657380. 00000080:00000010:2.1:1788201290.129827:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468760. 00010000:00000010:2.1:1788201290.345260:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7600. 00010000:00000010:2.1:1788201290.345268:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efa00. 00010000:00000010:2.1:1788201290.379662:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7900. 00010000:00000010:2.1:1788201290.379670:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef000. 00000080:00000010:2.1:1788201290.481447:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3778000. 00000080:00000010:2.1:1788201290.481456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc60. 00010000:00000010:2.1:1788201290.927219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6dc00. 00010000:00000010:2.1:1788201290.927228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182200. 00010000:00000010:2.1:1788201290.970307:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6dd00. 00010000:00000010:2.1:1788201290.970327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183600. 00000080:00000010:2.1:1788201290.975185:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e4d280. 00000080:00000010:2.1:1788201290.975196:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201d40. 00010000:00000010:2.1:1788201290.998522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b600. 00010000:00000010:2.1:1788201291.004389:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8be00. 00010000:00000010:2.1:1788201291.004479:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183000. 00000080:00000010:2.1:1788201291.004490:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e71080. 00000080:00000010:2.1:1788201291.004499:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468fe0. 00010000:00000010:2.1:1788201291.022136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b600. 00010000:00000010:2.1:1788201291.110179:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b500. 00010000:00000010:2.1:1788201291.110186:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183e00. 00000080:00000010:2.1:1788201291.110196:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab52ca40. 00000080:00000010:2.1:1788201291.110202:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ccc0. 00010000:00000010:2.1:1788201291.121514:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b400. 00010000:00000010:2.1:1788201291.331860:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6da00. 00010000:00000010:2.1:1788201291.331878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14a00. 00000080:00000010:2.1:1788201291.343489:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa10840. 00000080:00000010:2.1:1788201291.343507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e20. 00010000:00000010:2.1:1788201291.360026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b200. 00000080:00000010:2.1:1788201291.736671:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ddb9c0. 00000080:00000010:2.1:1788201291.736706:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c4c0. 00010000:00000010:2.1:1788201291.893159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7200. 00010000:00000010:2.1:1788201291.893169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90941200. 00010000:00000010:2.1:1788201292.241794:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4700. 00010000:00000010:2.1:1788201292.241801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481c00. 00000080:00000010:2.1:1788201292.251795:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3564200. 00000080:00000010:2.1:1788201292.251806:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c940. 00010000:00000010:2.1:1788201292.336054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480400. 00010000:00000010:2.1:1788201292.348591:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44100. 00010000:00000010:2.1:1788201292.348605:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481600. 00010000:00000010:2.1:1788201293.711147:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952200. 00010000:00000010:2.1:1788201293.711165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6800. 00010000:00000010:2.1:1788201293.748210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952300. 00010000:00000010:2.1:1788201293.748218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6200. 00000080:00000010:2.1:1788201293.791326:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf90840. 00000080:00000010:2.1:1788201293.791348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:2.1:1788201293.852177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bd00. 00010000:00000010:2.1:1788201293.852190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7e00. 00000080:00000010:2.1:1788201293.983365:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab73b180. 00000080:00000010:2.1:1788201293.983378:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201be0. 00010000:00000010:2.1:1788201293.991244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b200. 00010000:00000010:2.1:1788201294.093248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038f200. 00000080:00000010:2.1:1788201294.644514:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab759080. 00000080:00000010:2.1:1788201294.644536:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7c0. 00010000:00000010:2.1:1788201295.472384:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cc00. 00010000:00000010:2.1:1788201295.472395:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183200. 00000080:00000010:2.1:1788201295.494181:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab554200. 00000080:00000010:2.1:1788201295.494203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019c0. 00010000:00000010:2.1:1788201295.494212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b800. 00010000:00000010:2.1:1788201295.517301:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c200. 00010000:00000010:2.1:1788201295.517317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182a00. 00000080:00000010:2.1:1788201295.517327:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6bc200. 00000080:00000010:2.1:1788201295.517336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c20. 00010000:00000010:2.1:1788201295.529272:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b800. 00010000:00000010:2.1:1788201296.169477:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d300. 00010000:00000010:2.1:1788201296.169484:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4400. 00010000:00000010:2.1:1788201296.169487:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00000080:00000010:2.1:1788201296.181637:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa96c200. 00000080:00000010:2.1:1788201296.181656:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e00. 00010000:00000010:2.1:1788201296.223217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d000. 00010000:00000010:2.1:1788201296.223227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038f800. 00000080:00000010:2.1:1788201296.257811:0:407529:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37b3180. 00000080:00000010:2.1:1788201296.257823:0:407529:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686a0. 00010000:00000010:2.1:1788201296.366574:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183a00. 00000080:00000010:2.1:1788201296.425437:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa361e300. 00010000:00000010:2.1:1788201296.576283:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149d200. 00000080:00000010:2.1:1788201296.596180:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3570840. 00000080:00000010:2.1:1788201296.596197:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981722a0. 00010000:00000010:2.1:1788201297.416686:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7c00. 00010000:00000010:2.1:1788201297.416697:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00010000:00000010:2.1:1788201298.165180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c200. 00010000:00000010:2.1:1788201298.165196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4400. 00000080:00000010:2.1:1788201298.174809:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab787380. 00000080:00000010:2.1:1788201298.174830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:2.1:1788201298.185629:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5600. 00010000:00000010:2.1:1788201298.210178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cc00. 00010000:00000010:2.1:1788201298.210193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5e00. 00000080:00000010:2.1:1788201298.218505:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3524200. 00000080:00000010:2.1:1788201298.218518:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468060. 00010000:00000010:2.1:1788201298.228197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5600. 00010000:00000010:2.1:1788201298.979181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e000. 00010000:00000010:2.1:1788201298.979202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038f800. 00000080:00000010:2.1:1788201299.197165:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbecd280. 00000080:00000010:2.1:1788201299.197173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00010000:00000010:2.1:1788201299.355180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c100. 00010000:00000010:2.1:1788201299.355203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4a00. 00010000:00000010:2.1:1788201299.737262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e012100. 00010000:00000010:2.1:1788201299.737271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82800. 00010000:00000010:2.1:1788201299.861344:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e012000. 00010000:00000010:2.1:1788201299.861360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6600. 00000080:00000010:2.1:1788201299.886992:0:408120:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfef380. 00000080:00000010:2.1:1788201299.887040:0:408120:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014a0. 00010000:00000010:2.1:1788201300.734812:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50100. 00010000:00000010:2.1:1788201300.734824:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941d400. 00010000:00000010:2.1:1788201301.191300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50900. 00010000:00000010:2.1:1788201301.191318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149c200. 00000080:00000010:2.1:1788201301.215705:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe20000. 00000080:00000010:2.1:1788201301.215732:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:2.1:1788201301.229743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201301.377513:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50400. 00010000:00000010:2.1:1788201301.377537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149c000. 00000080:00000010:2.1:1788201301.391753:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e9ca40. 00000080:00000010:2.1:1788201301.391769:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b00. 00010000:00000010:2.1:1788201301.399204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00010000:00000010:2.1:1788201301.424331:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50500. 00010000:00000010:2.1:1788201301.424349:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182400. 00010000:00000010:2.1:1788201301.739815:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72200. 00010000:00000010:2.1:1788201301.739830:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183200. 00000080:00000010:2.1:1788201301.777403:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cdb180. 00000080:00000010:2.1:1788201301.777419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c760. 00010000:00000010:2.1:1788201301.783247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00010000:00000010:2.1:1788201301.802666:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24300. 00010000:00000010:2.1:1788201301.802686:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183600. 00000080:00000010:2.1:1788201301.818462:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5668840. 00000080:00000010:2.1:1788201301.818485:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894682c0. 00010000:00000010:2.1:1788201301.832036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00010000:00000010:2.1:1788201301.854675:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24200. 00010000:00000010:2.1:1788201301.854685:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183000. 00000080:00000010:2.1:1788201301.854695:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84063180. 00000080:00000010:2.1:1788201301.854702:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894681e0. 00000080:00000010:2.1:1788201301.956719:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa363b180. 00000080:00000010:2.1:1788201301.956749:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cec0. 00010000:00000010:2.1:1788201301.969036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871e00. 00010000:00000010:2.1:1788201302.004303:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50c00. 00010000:00000010:2.1:1788201302.004311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183e00. 00000080:00000010:2.1:1788201302.004318:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab586300. 00000080:00000010:2.1:1788201302.004324:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468300. 00000080:00000010:2.1:1788201302.576158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a80. 00010000:00000010:2.1:1788201302.808258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959600. 00010000:00000010:2.1:1788201302.808268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6800. 00010000:00000010:2.1:1788201303.124883:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870600. 00010000:00000010:2.1:1788201303.171244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201303.195200:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b900. 00010000:00000010:2.1:1788201303.195216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417600. 00000080:00000010:2.1:1788201303.355126:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5692940. 00000080:00000010:2.1:1788201303.355137:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2e0. 00010000:00000010:2.1:1788201303.402179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b700. 00010000:00000010:2.1:1788201303.402249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417a00. 00000080:00000010:2.1:1788201303.414354:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815b0000. 00000080:00000010:2.1:1788201303.414365:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468de0. 00010000:00000010:2.1:1788201303.420195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102000. 00010000:00000010:2.1:1788201303.830456:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103200. 00010000:00000010:2.1:1788201303.887524:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8be00. 00010000:00000010:2.1:1788201303.887541:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4400. 00000080:00000010:2.1:1788201303.908426:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc90840. 00000080:00000010:2.1:1788201303.908444:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201580. 00010000:00000010:2.1:1788201303.930237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103a00. 00010000:00000010:2.1:1788201303.980206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b300. 00010000:00000010:2.1:1788201303.980220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4a00. 00000080:00000010:2.1:1788201304.009148:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85953180. 00000080:00000010:2.1:1788201304.009166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468f40. 00010000:00000010:2.1:1788201304.022266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103400. 00010000:00000010:2.1:1788201304.063299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f100. 00010000:00000010:2.1:1788201304.063314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5400. 00000080:00000010:2.1:1788201304.097405:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84805ac0. 00000080:00000010:2.1:1788201304.097423:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894682c0. 00010000:00000010:2.1:1788201304.325399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6e00. 00010000:00000010:2.1:1788201304.325408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5800. 00010000:00000010:2.1:1788201304.813572:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f900. 00010000:00000010:2.1:1788201304.813579:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00000080:00000010:2.1:1788201304.813587:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817f2100. 00000080:00000010:2.1:1788201304.813593:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201400. 00010000:00000010:2.1:1788201304.817222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102800. 00000080:00000010:2.1:1788201305.079009:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f57380. 00000080:00000010:2.1:1788201305.079025:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52017c0. 00000080:00000010:2.1:1788201305.371229:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35bc200. 00000080:00000010:2.1:1788201305.371242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c020. 00010000:00000010:2.1:1788201305.551435:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7500. 00010000:00000010:2.1:1788201305.551467:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4000. 00000080:00000010:2.1:1788201305.563080:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8145eb40. 00000080:00000010:2.1:1788201305.563767:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c940. 00010000:00000010:2.1:1788201305.567867:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103400. 00010000:00000010:2.1:1788201305.795193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7600. 00010000:00000010:2.1:1788201305.795209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5800. 00010000:00000010:2.1:1788201306.129188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7300. 00010000:00000010:2.1:1788201306.129318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00000080:00000010:2.1:1788201306.154878:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815aca40. 00000080:00000010:2.1:1788201306.154907:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201820. 00010000:00000010:2.1:1788201306.317895:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51b00. 00010000:00000010:2.1:1788201306.317906:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4200. 00000080:00000010:2.1:1788201306.322527:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815ce300. 00000080:00000010:2.1:1788201306.322542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb00. 00010000:00000010:2.1:1788201306.334154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103000. 00000080:00000010:2.1:1788201306.837311:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5b2100. 00000080:00000010:2.1:1788201306.837332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c9e0. 00010000:00000010:2.1:1788201307.262283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e012a00. 00010000:00000010:2.1:1788201307.262302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416800. 00000080:00000010:2.1:1788201307.336154:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c14200. 00000080:00000010:2.1:1788201307.336175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cae0. 00010000:00000010:2.1:1788201307.368107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102800. 00010000:00000010:2.1:1788201307.604367:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e012300. 00010000:00000010:2.1:1788201307.604382:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653a000. 00010000:00000010:2.1:1788201308.513145:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44700. 00010000:00000010:2.1:1788201308.513159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217800. 00000080:00000010:2.1:1788201308.529133:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa377ca40. 00000080:00000010:2.1:1788201308.529148:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf20. 00010000:00000010:2.1:1788201308.534145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103e00. 00010000:00000010:2.1:1788201308.750151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577200. 00010000:00000010:2.1:1788201308.762183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952c00. 00010000:00000010:2.1:1788201308.762198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8200. 00000080:00000010:2.1:1788201308.780439:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbd95280. 00000080:00000010:2.1:1788201308.780449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce20. 00010000:00000010:2.1:1788201308.785742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103400. 00010000:00000010:2.1:1788201308.818189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952b00. 00010000:00000010:2.1:1788201308.818203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9400. 00000080:00000010:2.1:1788201308.818211:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7a5ac0. 00000080:00000010:2.1:1788201308.818218:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468900. 00010000:00000010:2.1:1788201308.822219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103400. 00010000:00000010:2.1:1788201308.857708:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952d00. 00010000:00000010:2.1:1788201308.857720:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8a00. 00010000:00000010:2.1:1788201309.170319:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44000. 00010000:00000010:2.1:1788201309.170334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9200. 00000080:00000010:2.1:1788201309.185292:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa342a100. 00000080:00000010:2.1:1788201309.185306:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201da0. 00010000:00000010:2.1:1788201309.718175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cd00. 00010000:00000010:2.1:1788201309.718189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24200. 00010000:00000010:2.1:1788201310.497069:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6dd00. 00010000:00000010:2.1:1788201310.497083:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83400. 00010000:00000010:2.1:1788201311.497968:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7d00. 00010000:00000010:2.1:1788201311.497984:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4800. 00000080:00000010:2.1:1788201311.517046:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56e7380. 00000080:00000010:2.1:1788201311.517063:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8c0. 00010000:00000010:2.1:1788201311.527177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:2.1:1788201311.561288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7000. 00010000:00000010:2.1:1788201311.561302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5a00. 00000080:00000010:2.1:1788201311.582227:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf33180. 00000080:00000010:2.1:1788201311.582244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468140. 00000080:00000010:2.1:1788201312.305502:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc87380. 00000080:00000010:2.1:1788201312.305516:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c80. 00010000:00000010:2.1:1788201312.312189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0800. 00010000:00000010:2.1:1788201312.462702:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7200. 00010000:00000010:2.1:1788201312.462718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b63e00. 00000080:00000010:2.1:1788201312.598167:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8db180. 00000080:00000010:2.1:1788201312.598186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172b60. 00010000:00000010:2.1:1788201312.606192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1a00. 00010000:00000010:2.1:1788201312.633680:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6900. 00010000:00000010:2.1:1788201312.633688:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00000080:00000010:2.1:1788201312.648232:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8f39c0. 00000080:00000010:2.1:1788201312.648247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468da0. 00010000:00000010:2.1:1788201312.648260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0800. 00010000:00000010:2.1:1788201312.686677:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51000. 00010000:00000010:2.1:1788201312.686691:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7600. 00000080:00000010:2.1:1788201312.686701:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54b1080. 00000080:00000010:2.1:1788201312.686709:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c20. 00010000:00000010:2.1:1788201312.695315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0800. 00010000:00000010:2.1:1788201313.640629:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124af00. 00010000:00000010:2.1:1788201313.640642:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576c00. 00000080:00000010:2.1:1788201314.091487:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa40840. 00000080:00000010:2.1:1788201314.091507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:2.1:1788201314.852215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84800. 00010000:00000010:2.1:1788201314.852230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14600. 00010000:00000010:2.1:1788201316.133670:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b900. 00010000:00000010:2.1:1788201316.133676:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33600. 00000080:00000010:2.1:1788201316.144157:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fc2100. 00000080:00000010:2.1:1788201316.144175:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f40. 00010000:00000010:2.1:1788201316.163259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00000080:00000010:2.1:1788201317.023221:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf71080. 00000080:00000010:2.1:1788201317.023237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c40. 00010000:00000010:2.1:1788201317.028160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1e00. 00000080:00000010:2.1:1788201318.748136:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c4c0. 00000080:00000010:2.1:1788201318.845131:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6b18c0. 00000080:00000010:2.1:1788201318.845143:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c020. 00010000:00000010:2.1:1788201319.567337:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f800. 00010000:00000010:2.1:1788201319.567352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82c00. 00000080:00000010:2.1:1788201319.579533:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7b0000. 00000080:00000010:2.1:1788201319.579550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c6e0. 00010000:00000010:2.1:1788201319.983238:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f000. 00010000:00000010:2.1:1788201319.983254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24200. 00000080:00000010:2.1:1788201320.117143:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbed7380. 00000080:00000010:2.1:1788201320.117160:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201740. 00000080:00000010:2.1:1788201320.228413:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5f6300. 00000080:00000010:2.1:1788201320.228435:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014a0. 00010000:00000010:2.1:1788201320.242555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1000. 00010000:00000010:2.1:1788201320.292166:0:407529:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7c00. 00010000:00000010:2.1:1788201320.292182:0:407529:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044c00. 00000080:00000010:2.1:1788201320.308674:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85880000. 00000080:00000010:2.1:1788201320.308687:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468740. 00010000:00000010:2.1:1788201320.447522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0400. 00000080:00000010:2.1:1788201320.874139:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbeaeb40. 00000080:00000010:2.1:1788201320.874153:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfc0. 00000080:00000010:2.1:1788201321.241575:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbd90000. 00000080:00000010:2.1:1788201321.241596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:2.1:1788201321.848630:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e012700. 00010000:00000010:2.1:1788201321.848645:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201322.977429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b100. 00010000:00000010:2.1:1788201322.977440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216600. 00000080:00000010:2.1:1788201322.985168:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56d6b40. 00000080:00000010:2.1:1788201322.985184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019c0. 00010000:00000010:2.1:1788201322.993360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0200. 00010000:00000010:2.1:1788201323.125173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86500. 00010000:00000010:2.1:1788201323.125450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33c00. 00010000:00000010:2.1:1788201323.260192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86e00. 00010000:00000010:2.1:1788201323.260204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33200. 00000080:00000010:2.1:1788201323.284255:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8595dac0. 00000080:00000010:2.1:1788201323.284273:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c400. 00000080:00000010:2.1:1788201323.443370:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa343b180. 00000080:00000010:2.1:1788201323.443389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201480. 00010000:00000010:2.1:1788201323.475262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1a00. 00010000:00000010:2.1:1788201323.830565:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952b00. 00010000:00000010:2.1:1788201323.830573:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044c00. 00010000:00000010:2.1:1788201323.868464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:2.1:1788201324.307366:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84e00. 00010000:00000010:2.1:1788201324.307382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24e00. 00000080:00000010:2.1:1788201324.319558:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9f4200. 00000080:00000010:2.1:1788201324.319588:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce20. 00010000:00000010:2.1:1788201324.521541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84800. 00010000:00000010:2.1:1788201324.521550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25800. 00000080:00000010:2.1:1788201324.528438:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3541080. 00000080:00000010:2.1:1788201324.528454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b60. 00010000:00000010:2.1:1788201324.532681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00010000:00000010:2.1:1788201325.423297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51900. 00010000:00000010:2.1:1788201325.423305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149c000. 00010000:00000010:2.1:1788201325.492462:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51400. 00010000:00000010:2.1:1788201325.492470:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6200. 00000080:00000010:2.1:1788201325.498232:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa368ca40. 00000080:00000010:2.1:1788201325.498247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00010000:00000010:2.1:1788201325.498257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:2.1:1788201325.743651:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aaac51300. 00010000:00000010:2.1:1788201325.743659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7000. 00010000:00000010:2.1:1788201325.932492:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d000. 00010000:00000010:2.1:1788201325.932504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416c00. 00000080:00000010:2.1:1788201325.940831:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa91dac0. 00000080:00000010:2.1:1788201325.940911:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c9e0. 00010000:00000010:2.1:1788201325.948472:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0400. 00010000:00000010:2.1:1788201325.967391:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6da00. 00010000:00000010:2.1:1788201325.967409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416e00. 00000080:00000010:2.1:1788201325.984519:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8164a100. 00000080:00000010:2.1:1788201325.984533:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468120. 00010000:00000010:2.1:1788201325.994257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0400. 00000080:00000010:2.1:1788201326.019149:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5688840. 00000080:00000010:2.1:1788201326.019168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468060. 00010000:00000010:2.1:1788201326.034360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00010000:00000010:2.1:1788201326.510950:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72a00. 00010000:00000010:2.1:1788201326.510960:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6400. 00000080:00000010:2.1:1788201326.525958:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cfb180. 00000080:00000010:2.1:1788201326.525978:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb00. 00010000:00000010:2.1:1788201326.533474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:2.1:1788201326.556787:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72400. 00010000:00000010:2.1:1788201326.556798:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6000. 00000080:00000010:2.1:1788201326.569380:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab505ac0. 00000080:00000010:2.1:1788201326.569393:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468860. 00010000:00000010:2.1:1788201326.569401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:2.1:1788201326.948226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6dd00. 00010000:00000010:2.1:1788201326.948242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c31e00. 00000080:00000010:2.1:1788201327.352456:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5dc3180. 00000080:00000010:2.1:1788201327.352473:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b00. 00010000:00000010:2.1:1788201327.372331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:2.1:1788201327.467321:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86300. 00010000:00000010:2.1:1788201327.467333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabae00. 00010000:00000010:2.1:1788201327.616494:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86400. 00010000:00000010:2.1:1788201327.616501:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba400. 00010000:00000010:2.1:1788201327.732322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72c00. 00010000:00000010:2.1:1788201327.732337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33000. 00010000:00000010:2.1:1788201328.371665:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b600. 00010000:00000010:2.1:1788201328.371680:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:2.1:1788201328.755178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b800. 00010000:00000010:2.1:1788201328.755194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:2.1:1788201330.102141:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96fa00. 00010000:00000010:2.1:1788201330.102187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00010000:00000010:2.1:1788201330.398177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96fc00. 00010000:00000010:2.1:1788201330.398190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1a00. 00010000:00000010:2.1:1788201330.415379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e200. 00010000:00000010:2.1:1788201330.449184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f200. 00010000:00000010:2.1:1788201330.449192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1400. 00010000:00000010:2.1:1788201330.493130:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f900. 00010000:00000010:2.1:1788201330.493137:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00000080:00000010:2.1:1788201330.498317:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4b98c0. 00000080:00000010:2.1:1788201330.498339:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c040. 00010000:00000010:2.1:1788201330.516134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00000080:00000010:2.1:1788201331.035149:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f0d280. 00000080:00000010:2.1:1788201331.035166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201400. 00010000:00000010:2.1:1788201331.157288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72a00. 00010000:00000010:2.1:1788201331.157302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6200. 00000080:00000010:2.1:1788201331.171791:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56fa100. 00000080:00000010:2.1:1788201331.171816:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a80. 00010000:00000010:2.1:1788201331.176148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83200. 00010000:00000010:2.1:1788201331.211605:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72700. 00010000:00000010:2.1:1788201331.211624:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15c00. 00000080:00000010:2.1:1788201331.234885:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34a0840. 00000080:00000010:2.1:1788201331.234902:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468da0. 00010000:00000010:2.1:1788201331.899166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7600. 00010000:00000010:2.1:1788201331.899180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8c00. 00000080:00000010:2.1:1788201332.063765:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf1d280. 00000080:00000010:2.1:1788201332.063786:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00000080:00000010:2.1:1788201332.826377:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf6dac0. 00000080:00000010:2.1:1788201332.826387:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00010000:00000010:2.1:1788201333.048004:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6638d00. 00010000:00000010:2.1:1788201333.048014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00000080:00000010:2.1:1788201333.057223:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcc8000. 00000080:00000010:2.1:1788201333.057241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb00. 00010000:00000010:2.1:1788201333.061726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:2.1:1788201333.217296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6638400. 00010000:00000010:2.1:1788201333.217307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038fc00. 00000080:00000010:2.1:1788201333.217316:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8156c200. 00000080:00000010:2.1:1788201333.217323:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8e0. 00010000:00000010:2.1:1788201333.455196:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb500. 00010000:00000010:2.1:1788201333.455209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4800. 00000080:00000010:2.1:1788201333.455219:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84134a40. 00000080:00000010:2.1:1788201333.455228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf60. 00010000:00000010:2.1:1788201333.482149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14800. 00010000:00000010:2.1:1788201333.524647:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbd00. 00010000:00000010:2.1:1788201333.524660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5800. 00000080:00000010:2.1:1788201333.542616:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5432100. 00000080:00000010:2.1:1788201333.542637:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468200. 00010000:00000010:2.1:1788201333.543632:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103800. 00010000:00000010:2.1:1788201333.829194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50600. 00010000:00000010:2.1:1788201333.829202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5200. 00010000:00000010:2.1:1788201333.862142:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50f00. 00010000:00000010:2.1:1788201333.862158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4000. 00000080:00000010:2.1:1788201333.975160:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814a2100. 00000080:00000010:2.1:1788201333.975180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cae0. 00010000:00000010:2.1:1788201334.080602:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb900. 00010000:00000010:2.1:1788201334.080624:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3400. 00000080:00000010:2.1:1788201334.346412:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4ddac0. 00000080:00000010:2.1:1788201334.346428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c440. 00010000:00000010:2.1:1788201334.382136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577200. 00010000:00000010:2.1:1788201334.410429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb800. 00010000:00000010:2.1:1788201334.410439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb200. 00000080:00000010:2.1:1788201334.410450:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3546b40. 00000080:00000010:2.1:1788201334.410463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:2.1:1788201334.414303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15000. 00010000:00000010:2.1:1788201334.433171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb000. 00010000:00000010:2.1:1788201334.433179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb600. 00000080:00000010:2.1:1788201334.441213:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814bc200. 00000080:00000010:2.1:1788201334.441228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468060. 00010000:00000010:2.1:1788201334.447653:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15000. 00010000:00000010:2.1:1788201334.528441:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952e00. 00010000:00000010:2.1:1788201334.528451:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabbc00. 00000080:00000010:2.1:1788201334.537232:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa43180. 00000080:00000010:2.1:1788201334.537244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b80. 00010000:00000010:2.1:1788201334.545152:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14a00. 00010000:00000010:2.1:1788201335.372475:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7900. 00010000:00000010:2.1:1788201335.372481:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217200. 00010000:00000010:2.1:1788201335.443584:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7000. 00010000:00000010:2.1:1788201336.358131:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4c00. 00010000:00000010:2.1:1788201336.378878:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24c00. 00010000:00000010:2.1:1788201336.378885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62400. 00000080:00000010:2.1:1788201336.383040:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc8d280. 00000080:00000010:2.1:1788201336.383057:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e00. 00010000:00000010:2.1:1788201336.401249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103a00. 00010000:00000010:2.1:1788201336.677184:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7b00. 00010000:00000010:2.1:1788201336.677193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3200. 00000080:00000010:2.1:1788201336.682733:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fe3180. 00000080:00000010:2.1:1788201336.682752:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc40. 00010000:00000010:2.1:1788201336.696133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4600. 00010000:00000010:2.1:1788201337.395132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5600. 00010000:00000010:2.1:1788201337.561253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3c00. 00010000:00000010:2.1:1788201337.561268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:2.1:1788201337.969248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6638400. 00010000:00000010:2.1:1788201337.969256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14600. 00000080:00000010:2.1:1788201337.974162:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa933180. 00000080:00000010:2.1:1788201337.974181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201760. 00010000:00000010:2.1:1788201337.983123:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4a00. 00000080:00000010:2.1:1788201338.525142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fe7380. 00000080:00000010:2.1:1788201338.525217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8c0. 00010000:00000010:2.1:1788201338.540184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201338.681520:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6638900. 00010000:00000010:2.1:1788201338.681535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83be00. 00010000:00000010:2.1:1788201339.116644:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952f00. 00010000:00000010:2.1:1788201339.116659:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83b000. 00010000:00000010:2.1:1788201339.309855:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952300. 00010000:00000010:2.1:1788201339.309878:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5400. 00000080:00000010:2.1:1788201339.326352:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85941080. 00000080:00000010:2.1:1788201339.326370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8e0. 00010000:00000010:2.1:1788201339.338232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00000080:00000010:2.1:1788201339.466727:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7439c0. 00000080:00000010:2.1:1788201339.466744:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201be0. 00010000:00000010:2.1:1788201339.488388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eee00. 00010000:00000010:2.1:1788201339.563199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124ab00. 00010000:00000010:2.1:1788201339.563210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9c00. 00000080:00000010:2.1:1788201339.570262:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d82100. 00000080:00000010:2.1:1788201339.570278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c6e0. 00010000:00000010:2.1:1788201339.655592:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124af00. 00010000:00000010:2.1:1788201339.655603:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8400. 00000080:00000010:2.1:1788201339.673116:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5670000. 00000080:00000010:2.1:1788201339.673131:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00000080:00000010:2.1:1788201339.748154:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3695ac0. 00000080:00000010:2.1:1788201339.748177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cec0. 00010000:00000010:2.1:1788201339.764134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417400. 00010000:00000010:2.1:1788201339.787936:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952000. 00010000:00000010:2.1:1788201339.787955:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8e00. 00000080:00000010:2.1:1788201339.805163:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3774a40. 00000080:00000010:2.1:1788201339.805181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468900. 00010000:00000010:2.1:1788201339.821142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417400. 00010000:00000010:2.1:1788201339.951696:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952d00. 00010000:00000010:2.1:1788201339.951709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9000. 00010000:00000010:2.1:1788201340.281270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86400. 00010000:00000010:2.1:1788201340.281283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06000. 00000080:00000010:2.1:1788201340.295496:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5656b40. 00000080:00000010:2.1:1788201340.295515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c6c0. 00010000:00000010:2.1:1788201340.304372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416400. 00000080:00000010:2.1:1788201340.693143:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6ca940. 00000080:00000010:2.1:1788201340.693155:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:2.1:1788201340.785168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86c00. 00010000:00000010:2.1:1788201340.785176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba600. 00000080:00000010:2.1:1788201340.793172:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf16300. 00000080:00000010:2.1:1788201340.793189:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:2.1:1788201340.799360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417200. 00010000:00000010:2.1:1788201340.817171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86000. 00010000:00000010:2.1:1788201340.817184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba400. 00000080:00000010:2.1:1788201340.829399:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6b9080. 00000080:00000010:2.1:1788201340.829417:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686c0. 00010000:00000010:2.1:1788201341.949704:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a9a96f200. 00010000:00000010:2.1:1788201341.949711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00010000:00000010:2.1:1788201342.054958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06e00. 00010000:00000010:2.1:1788201342.510321:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84900. 00010000:00000010:2.1:1788201342.510334:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8200. 00010000:00000010:2.1:1788201342.866182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6800. 00010000:00000010:2.1:1788201342.971296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4d00. 00010000:00000010:2.1:1788201342.971310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1a00. 00010000:00000010:2.1:1788201344.268349:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039100. 00010000:00000010:2.1:1788201344.268358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb600. 00000080:00000010:2.1:1788201344.274566:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e3eb40. 00000080:00000010:2.1:1788201344.274579:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c020. 00010000:00000010:2.1:1788201344.281299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417400. 00010000:00000010:2.1:1788201344.313451:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039400. 00010000:00000010:2.1:1788201344.313459:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabae00. 00000080:00000010:2.1:1788201344.320312:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35e3180. 00000080:00000010:2.1:1788201344.320327:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468860. 00010000:00000010:2.1:1788201344.332238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416800. 00010000:00000010:2.1:1788201344.439540:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6de00. 00010000:00000010:2.1:1788201344.439556:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb800. 00010000:00000010:2.1:1788201344.590988:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d200. 00010000:00000010:2.1:1788201344.590998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabac00. 00000080:00000010:2.1:1788201344.739978:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa98b180. 00000080:00000010:2.1:1788201344.739993:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00000080:00000010:2.1:1788201345.138743:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab757380. 00000080:00000010:2.1:1788201345.139064:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201640. 00010000:00000010:2.1:1788201345.299305:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:2.1:1788201345.299321:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14400. 00010000:00000010:2.1:1788201345.299326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24400. 00010000:00000010:2.1:1788201345.299333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6800. 00010000:00000010:2.1:1788201345.299336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00010000:00000010:2.1:1788201345.299340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b63800. 00010000:00000010:2.1:1788201345.299344:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577600. 00010000:00000010:2.1:1788201345.299347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4000. 00010000:00000010:2.1:1788201345.299350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6c00. 00010000:00000010:2.1:1788201345.299353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7e00. 00010000:00000010:2.1:1788201345.299357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577c00. 00010000:00000010:2.1:1788201345.299360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:2.1:1788201345.299366:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba000. 00010000:00000010:2.1:1788201345.299371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653be00. 00010000:00000010:2.1:1788201345.299374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871c00. 00010000:00000010:2.1:1788201345.299378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32800. 00010000:00000010:2.1:1788201345.299385:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3600. 00010000:00000010:2.1:1788201345.299388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417e00. 00010000:00000010:2.1:1788201345.299391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653bc00. 00010000:00000010:2.1:1788201345.299394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103600. 00010000:00000010:2.1:1788201345.299399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33a00. 00010000:00000010:2.1:1788201345.299402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15800. 00010000:00000010:2.1:1788201345.299406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30a00. 00010000:00000010:2.1:1788201345.299506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8000. 00010000:00000010:2.1:1788201345.299512:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941c400. 00010000:00000010:2.1:1788201345.299515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00010000:00000010:2.1:1788201345.299518:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416600. 00010000:00000010:2.1:1788201345.299529:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952600. 00010000:00000010:2.1:1788201345.299627:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5600. 00010000:00000010:2.1:1788201345.943420:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959600. 00010000:00000010:2.1:1788201345.943436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103200. 00000080:00000010:2.1:1788201345.974152:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34a3180. 00000080:00000010:2.1:1788201345.974168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a80. 00010000:00000010:2.1:1788201345.985268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481800. 00010000:00000010:2.1:1788201346.011897:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959000. 00010000:00000010:2.1:1788201346.011918:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103600. 00000080:00000010:2.1:1788201346.026896:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85969080. 00000080:00000010:2.1:1788201346.026914:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468f60. 00000080:00000010:2.1:1788201346.282561:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8159d280. 00000080:00000010:2.1:1788201346.282576:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201ae0. 00010000:00000010:2.1:1788201346.282584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481c00. 00000080:00000010:2.1:1788201346.532236:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5b18c0. 00000080:00000010:2.1:1788201346.532253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015e0. 00010000:00000010:2.1:1788201346.547291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481000. 00010000:00000010:2.1:1788201346.991320:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e600. 00010000:00000010:2.1:1788201346.991335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149d200. 00000080:00000010:2.1:1788201347.017495:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab63eb40. 00000080:00000010:2.1:1788201347.017516:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:2.1:1788201347.027873:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481600. 00010000:00000010:2.1:1788201348.402522:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6400. 00010000:00000010:2.1:1788201348.902766:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124a900. 00010000:00000010:2.1:1788201348.902777:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653a400. 00000080:00000010:2.1:1788201348.915999:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54d7380. 00000080:00000010:2.1:1788201348.916027:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b80. 00010000:00000010:2.1:1788201349.119229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124ab00. 00010000:00000010:2.1:1788201349.119239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6800. 00010000:00000010:2.1:1788201349.476376:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e400. 00010000:00000010:2.1:1788201349.476389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941d400. 00000080:00000010:2.1:1788201349.484294:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b840639c0. 00000080:00000010:2.1:1788201349.484322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019c0. 00010000:00000010:2.1:1788201349.501154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480800. 00010000:00000010:2.1:1788201349.520297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e500. 00010000:00000010:2.1:1788201349.520311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941d600. 00000080:00000010:2.1:1788201349.545197:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f5a100. 00000080:00000010:2.1:1788201349.545210:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468120. 00010000:00000010:2.1:1788201349.566270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480800. 00010000:00000010:2.1:1788201349.796309:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6dc00. 00010000:00000010:2.1:1788201349.796325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941cc00. 00000080:00000010:2.1:1788201349.809522:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3548000. 00000080:00000010:2.1:1788201349.809538:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:2.1:1788201349.816920:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480200. 00010000:00000010:2.1:1788201349.896152:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6df00. 00010000:00000010:2.1:1788201349.896162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941ce00. 00010000:00000010:2.1:1788201349.900271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7400. 00010000:00000010:2.1:1788201349.935218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d300. 00010000:00000010:2.1:1788201349.935236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941de00. 00000080:00000010:2.1:1788201349.963525:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8413a940. 00000080:00000010:2.1:1788201349.963549:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201700. 00010000:00000010:2.1:1788201349.975236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480800. 00010000:00000010:2.1:1788201350.021224:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d500. 00010000:00000010:2.1:1788201350.021241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941c000. 00010000:00000010:2.1:1788201350.266465:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d200. 00010000:00000010:2.1:1788201350.266483:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6540c00. 00000080:00000010:2.1:1788201350.278705:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc9eb40. 00000080:00000010:2.1:1788201350.278718:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018c0. 00010000:00000010:2.1:1788201350.287162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481000. 00010000:00000010:2.1:1788201350.336568:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d900. 00010000:00000010:2.1:1788201350.336584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6541800. 00000080:00000010:2.1:1788201350.349272:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa344d280. 00000080:00000010:2.1:1788201350.349289:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468560. 00010000:00000010:2.1:1788201350.353258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481000. 00010000:00000010:2.1:1788201351.513498:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959f00. 00010000:00000010:2.1:1788201351.513514:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06c00. 00010000:00000010:2.1:1788201351.651387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86a00. 00010000:00000010:2.1:1788201351.651406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07000. 00010000:00000010:2.1:1788201351.697472:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86000. 00010000:00000010:2.1:1788201351.697478:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06a00. 00010000:00000010:2.1:1788201351.767148:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039100. 00010000:00000010:2.1:1788201351.767167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07800. 00010000:00000010:2.1:1788201351.942278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039500. 00010000:00000010:2.1:1788201351.942294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabba00. 00000080:00000010:2.1:1788201351.986917:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5a6300. 00000080:00000010:2.1:1788201351.986936:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172960. 00010000:00000010:2.1:1788201352.005775:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039600. 00010000:00000010:2.1:1788201352.005784:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabbc00. 00010000:00000010:2.1:1788201352.248664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039000. 00010000:00000010:2.1:1788201352.248672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9600. 00000080:00000010:2.1:1788201353.085337:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab58dac0. 00000080:00000010:2.1:1788201353.085348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b60. 00010000:00000010:2.1:1788201353.104124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480400. 00010000:00000010:2.1:1788201353.124149:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124a900. 00010000:00000010:2.1:1788201353.124155:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941c600. 00000080:00000010:2.1:1788201353.129532:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84a1a100. 00000080:00000010:2.1:1788201353.129543:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468900. 00010000:00000010:2.1:1788201353.135247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480400. 00010000:00000010:2.1:1788201353.165949:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124a800. 00010000:00000010:2.1:1788201353.166021:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941c400. 00000080:00000010:2.1:1788201353.175142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3552100. 00000080:00000010:2.1:1788201353.175153:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468f00. 00010000:00000010:2.1:1788201353.370980:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22f00. 00010000:00000010:2.1:1788201353.370987:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6600. 00000080:00000010:2.1:1788201353.378150:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa567c200. 00000080:00000010:2.1:1788201353.378173:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e40. 00010000:00000010:2.1:1788201353.392418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480600. 00010000:00000010:2.1:1788201353.525466:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22900. 00010000:00000010:2.1:1788201353.525484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653ae00. 00010000:00000010:2.1:1788201354.057803:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24c00. 00010000:00000010:2.1:1788201354.057812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25600. 00000080:00000010:2.1:1788201354.064541:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eceb40. 00000080:00000010:2.1:1788201354.064556:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8e0. 00010000:00000010:2.1:1788201354.068308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481200. 00010000:00000010:2.1:1788201354.134189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24d00. 00010000:00000010:2.1:1788201354.134205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24000. 00000080:00000010:2.1:1788201354.148857:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34e5280. 00000080:00000010:2.1:1788201354.148876:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc00. 00010000:00000010:2.1:1788201354.156259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481200. 00010000:00000010:2.1:1788201354.834513:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959e00. 00010000:00000010:2.1:1788201354.834526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416400. 00010000:00000010:2.1:1788201354.909289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24200. 00010000:00000010:2.1:1788201354.909300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417200. 00010000:00000010:2.1:1788201355.087145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6600. 00010000:00000010:2.1:1788201355.118175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e600. 00010000:00000010:2.1:1788201355.118191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416c00. 00000080:00000010:2.1:1788201355.225169:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cb39c0. 00000080:00000010:2.1:1788201355.225191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172200. 00010000:00000010:2.1:1788201355.236161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480e00. 00000080:00000010:2.1:1788201355.417249:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9a0840. 00000080:00000010:2.1:1788201355.417301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201aa0. 00010000:00000010:2.1:1788201355.855254:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c600. 00010000:00000010:2.1:1788201355.855269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4a00. 00000080:00000010:2.1:1788201355.866176:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34c8840. 00000080:00000010:2.1:1788201355.866201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:2.1:1788201355.879150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480200. 00010000:00000010:2.1:1788201356.366198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:2.1:1788201356.366215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a200. 00010000:00000010:2.1:1788201356.576192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c900. 00010000:00000010:2.1:1788201356.576208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83ac00. 00010000:00000010:2.1:1788201356.672189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c300. 00010000:00000010:2.1:1788201356.672202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83aa00. 00010000:00000010:2.1:1788201357.644134:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84800. 00010000:00000010:2.1:1788201357.644153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07800. 00000080:00000010:2.1:1788201357.668133:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54498c0. 00000080:00000010:2.1:1788201357.668148:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201580. 00010000:00000010:2.1:1788201357.836550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32a00. 00000080:00000010:2.1:1788201358.781188:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9af380. 00000080:00000010:2.1:1788201358.781202:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201540. 00010000:00000010:2.1:1788201358.800141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3e00. 00000080:00000010:2.1:1788201359.156280:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa369eb40. 00000080:00000010:2.1:1788201359.156296:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00000080:00000010:2.1:1788201359.797650:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35898c0. 00000080:00000010:2.1:1788201359.797661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c020. 00010000:00000010:2.1:1788201359.879181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7200. 00010000:00000010:2.1:1788201359.879188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481c00. 00010000:00000010:2.1:1788201360.089159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952300. 00010000:00000010:2.1:1788201360.089173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480e00. 00010000:00000010:2.1:1788201360.162085:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952000. 00010000:00000010:2.1:1788201360.162115:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481000. 00010000:00000010:2.1:1788201360.204194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952100. 00010000:00000010:2.1:1788201360.204207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481e00. 00010000:00000010:2.1:1788201360.674357:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b93959300. 00010000:00000010:2.1:1788201360.674374:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9c00. 00000080:00000010:2.1:1788201360.702184:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6e8000. 00000080:00000010:2.1:1788201360.702197:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc00. 00010000:00000010:2.1:1788201360.711334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00000080:00000010:2.1:1788201360.792446:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbdd8000. 00000080:00000010:2.1:1788201360.792467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201820. 00010000:00000010:2.1:1788201362.037204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cc00. 00010000:00000010:2.1:1788201362.037216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4a00. 00010000:00000010:2.1:1788201362.411150:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6700. 00010000:00000010:2.1:1788201362.411156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3600. 00010000:00000010:2.1:1788201362.864818:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6000. 00010000:00000010:2.1:1788201362.864825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33a00. 00010000:00000010:2.1:1788201363.372341:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86300. 00010000:00000010:2.1:1788201363.372354:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00000080:00000010:2.1:1788201363.386319:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf2a100. 00000080:00000010:2.1:1788201363.386340:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019c0. 00010000:00000010:2.1:1788201363.478148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2a00. 00010000:00000010:2.1:1788201363.899145:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7b00. 00010000:00000010:2.1:1788201363.899164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00000080:00000010:2.1:1788201364.108643:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5239c0. 00000080:00000010:2.1:1788201364.108658:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:2.1:1788201364.312265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86a00. 00010000:00000010:2.1:1788201364.312276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7800. 00010000:00000010:2.1:1788201364.848165:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50a00. 00010000:00000010:2.1:1788201364.848173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e600. 00010000:00000010:2.1:1788201364.948755:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50200. 00010000:00000010:2.1:1788201364.948762:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90941000. 00000080:00000010:2.1:1788201364.957132:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa349b180. 00000080:00000010:2.1:1788201364.957143:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201900. 00010000:00000010:2.1:1788201364.961142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:2.1:1788201364.992178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039300. 00010000:00000010:2.1:1788201364.992187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90940400. 00010000:00000010:2.1:1788201365.085994:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039800. 00010000:00000010:2.1:1788201365.086005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90941400. 00000080:00000010:2.1:1788201365.093137:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35f1080. 00000080:00000010:2.1:1788201365.093149:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201480. 00010000:00000010:2.1:1788201365.101531:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00010000:00000010:2.1:1788201365.117263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039c00. 00010000:00000010:2.1:1788201365.117276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90940600. 00000080:00000010:2.1:1788201365.126027:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8156a100. 00000080:00000010:2.1:1788201365.126054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468960. 00010000:00000010:2.1:1788201365.132644:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00010000:00000010:2.1:1788201366.291312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039900. 00010000:00000010:2.1:1788201366.291326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df2400. 00010000:00000010:2.1:1788201366.332491:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24600. 00010000:00000010:2.1:1788201366.332502:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7400. 00010000:00000010:2.1:1788201367.037179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24d00. 00010000:00000010:2.1:1788201367.037192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9200. 00000080:00000010:2.1:1788201367.046669:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8155ca40. 00000080:00000010:2.1:1788201367.046688:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:2.1:1788201367.073356:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:2.1:1788201368.094201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7500. 00010000:00000010:2.1:1788201368.094220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabba00. 00000080:00000010:2.1:1788201368.109782:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab76e300. 00000080:00000010:2.1:1788201368.109802:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:2.1:1788201368.127726:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00000080:00000010:2.1:1788201368.408366:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8482f380. 00000080:00000010:2.1:1788201368.408379:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cec0. 00010000:00000010:2.1:1788201368.420014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:2.1:1788201368.449183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7000. 00010000:00000010:2.1:1788201368.459133:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb600. 00010000:00000010:2.1:1788201368.674648:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4200. 00010000:00000010:2.1:1788201368.674663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba400. 00000080:00000010:2.1:1788201368.674671:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5702940. 00000080:00000010:2.1:1788201368.674680:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015e0. 00010000:00000010:2.1:1788201368.699588:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3400. 00010000:00000010:2.1:1788201368.725217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4700. 00010000:00000010:2.1:1788201368.725233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb416800. 00000080:00000010:2.1:1788201368.732262:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe48000. 00000080:00000010:2.1:1788201368.732294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468500. 00010000:00000010:2.1:1788201368.764743:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2400. 00010000:00000010:2.1:1788201369.169978:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7f00. 00010000:00000010:2.1:1788201369.169988:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417600. 00010000:00000010:2.1:1788201369.468829:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bf00. 00010000:00000010:2.1:1788201369.468835:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030800. 00000080:00000010:2.1:1788201370.444738:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa884a40. 00000080:00000010:2.1:1788201370.444756:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f20. 00010000:00000010:2.1:1788201370.465224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2400. 00010000:00000010:2.1:1788201370.503473:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039b00. 00010000:00000010:2.1:1788201370.503488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33200. 00000080:00000010:2.1:1788201370.503496:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85825280. 00000080:00000010:2.1:1788201370.503504:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468de0. 00010000:00000010:2.1:1788201370.516142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2a00. 00010000:00000010:2.1:1788201370.546231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039500. 00010000:00000010:2.1:1788201370.546248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32000. 00010000:00000010:2.1:1788201370.687173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039c00. 00010000:00000010:2.1:1788201370.687182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32600. 00000080:00000010:2.1:1788201370.687192:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815a39c0. 00000080:00000010:2.1:1788201370.687200:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:2.1:1788201370.700882:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:2.1:1788201370.710077:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039800. 00010000:00000010:2.1:1788201370.710108:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32800. 00000080:00000010:2.1:1788201370.720836:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817fc200. 00000080:00000010:2.1:1788201370.720848:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468b60. 00000080:00000010:2.1:1788201372.011148:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa351d280. 00000080:00000010:2.1:1788201372.011168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca60. 00010000:00000010:2.1:1788201372.068681:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86900. 00010000:00000010:2.1:1788201372.068687:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a800. 00000080:00000010:2.1:1788201372.076567:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5403180. 00000080:00000010:2.1:1788201372.076587:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2e0. 00000080:00000010:2.1:1788201372.405155:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7ad280. 00000080:00000010:2.1:1788201372.405174:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201cc0. 00010000:00000010:2.1:1788201373.229183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952700. 00010000:00000010:2.1:1788201373.229195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9400. 00010000:00000010:2.1:1788201373.477685:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e300. 00010000:00000010:2.1:1788201373.477694:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9600. 00000080:00000010:2.1:1788201373.494148:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54739c0. 00000080:00000010:2.1:1788201373.494165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c00. 00010000:00000010:2.1:1788201373.593528:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec500. 00010000:00000010:2.1:1788201373.593538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038f600. 00000080:00000010:2.1:1788201373.599579:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5682100. 00000080:00000010:2.1:1788201373.599594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:2.1:1788201373.606124:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3000. 00010000:00000010:2.1:1788201373.622177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecc00. 00010000:00000010:2.1:1788201373.622192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e600. 00000080:00000010:2.1:1788201373.635678:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f5f380. 00000080:00000010:2.1:1788201373.635704:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686c0. 00010000:00000010:2.1:1788201373.684592:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3000. 00010000:00000010:2.1:1788201373.999151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a000. 00010000:00000010:2.1:1788201374.100397:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22d00. 00010000:00000010:2.1:1788201374.100417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6e00. 00010000:00000010:2.1:1788201374.495161:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:2.1:1788201374.495168:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7800. 00000080:00000010:2.1:1788201374.499252:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814d6b40. 00000080:00000010:2.1:1788201374.499281:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014a0. 00010000:00000010:2.1:1788201374.517301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2c00. 00010000:00000010:2.1:1788201375.071219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737df00. 00010000:00000010:2.1:1788201375.071238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3200. 00000080:00000010:2.1:1788201375.106137:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcddac0. 00000080:00000010:2.1:1788201375.106156:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c9e0. 00010000:00000010:2.1:1788201375.109256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3400. 00010000:00000010:2.1:1788201376.131595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4700. 00010000:00000010:2.1:1788201376.131605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5e00. 00010000:00000010:2.1:1788201376.544137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0200. 00000080:00000010:2.1:1788201376.962598:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81595ac0. 00000080:00000010:2.1:1788201376.962616:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7c0. 00010000:00000010:2.1:1788201376.965568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3e00. 00000080:00000010:2.1:1788201377.138615:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab74e300. 00000080:00000010:2.1:1788201377.138633:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cbc0. 00000080:00000010:2.1:1788201377.194642:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe62940. 00000080:00000010:2.1:1788201377.194672:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894680a0. 00010000:00000010:2.1:1788201377.820395:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86000. 00010000:00000010:2.1:1788201377.820421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00000080:00000010:2.1:1788201377.820432:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab506b40. 00000080:00000010:2.1:1788201377.820441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201900. 00010000:00000010:2.1:1788201377.832660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00010000:00000010:2.1:1788201377.867787:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86c00. 00010000:00000010:2.1:1788201377.867796:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577c00. 00000080:00000010:2.1:1788201377.875479:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f63180. 00000080:00000010:2.1:1788201377.875493:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ec0. 00010000:00000010:2.1:1788201377.879520:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00010000:00000010:2.1:1788201377.901862:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86e00. 00010000:00000010:2.1:1788201377.901871:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577600. 00000080:00000010:2.1:1788201377.908659:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5440840. 00000080:00000010:2.1:1788201377.908683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ba0. 00010000:00000010:2.1:1788201377.925546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00000080:00000010:2.1:1788201379.118240:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57c2100. 00000080:00000010:2.1:1788201379.118260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b60. 00010000:00000010:2.1:1788201379.286202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb748df00. 00010000:00000010:2.1:1788201379.286211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab83a000. 00000080:00000010:2.1:1788201379.296143:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8147dac0. 00000080:00000010:2.1:1788201379.296153:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201fe0. 00010000:00000010:2.1:1788201379.594395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb000. 00010000:00000010:2.1:1788201379.738962:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00010000:00000010:2.1:1788201379.907962:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:2.1:1788201379.907971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3200. 00000080:00000010:2.1:1788201379.907980:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e1eb40. 00000080:00000010:2.1:1788201379.907987:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:2.1:1788201380.401281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d09600. 00010000:00000010:2.1:1788201380.401295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e600. 00000080:00000010:2.1:1788201380.414047:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81454a40. 00000080:00000010:2.1:1788201380.414064:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00010000:00000010:2.1:1788201380.424160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2e00. 00010000:00000010:2.1:1788201380.798272:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ece00. 00010000:00000010:2.1:1788201380.798280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9a00. 00010000:00000010:2.1:1788201381.944244:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952b00. 00010000:00000010:2.1:1788201381.944250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24400. 00010000:00000010:2.1:1788201382.172394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952000. 00010000:00000010:2.1:1788201382.172408:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee33a00. 00000080:00000010:2.1:1788201382.193260:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814b7380. 00000080:00000010:2.1:1788201382.193277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:2.1:1788201382.193290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2a00. 00000080:00000010:2.1:1788201382.298774:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814f5ac0. 00000080:00000010:2.1:1788201382.298787:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201640. 00010000:00000010:2.1:1788201382.304795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2400. 00010000:00000010:2.1:1788201382.650248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d09700. 00010000:00000010:2.1:1788201382.650261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaee32a00. 00000080:00000010:2.1:1788201382.662131:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3706300. 00000080:00000010:2.1:1788201382.662143:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201200. 00010000:00000010:2.1:1788201382.668452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:2.1:1788201382.937561:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb748d100. 00010000:00000010:2.1:1788201382.937573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481a00. 00000080:00000010:2.1:1788201382.951326:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5799080. 00000080:00000010:2.1:1788201382.951341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb00. 00010000:00000010:2.1:1788201382.960697:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:2.1:1788201382.972716:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb748db00. 00010000:00000010:2.1:1788201382.972729:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480400. 00010000:00000010:2.1:1788201383.179829:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb748d500. 00010000:00000010:2.1:1788201383.179843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481200. 00000080:00000010:2.1:1788201383.194260:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ebb9c0. 00000080:00000010:2.1:1788201383.194277:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201400. 00010000:00000010:2.1:1788201383.211363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2800. 00010000:00000010:2.1:1788201383.279321:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22700. 00010000:00000010:2.1:1788201383.279335:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481000. 00000080:00000010:2.1:1788201383.289854:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fa6b40. 00000080:00000010:2.1:1788201383.289867:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c4c0. 00010000:00000010:2.1:1788201383.300968:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2800. 00010000:00000010:2.1:1788201383.333172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22400. 00010000:00000010:2.1:1788201383.333184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481e00. 00000080:00000010:2.1:1788201383.356307:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b98905ac0. 00000080:00000010:2.1:1788201383.356326:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468860. 00010000:00000010:2.1:1788201383.374148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2800. 00000080:00000010:2.1:1788201383.510143:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815d0840. 00000080:00000010:2.1:1788201383.510158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c6e0. 00010000:00000010:2.1:1788201383.736269:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22200. 00010000:00000010:2.1:1788201383.736275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149d800. 00000080:00000010:2.1:1788201383.974581:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8406c200. 00000080:00000010:2.1:1788201383.974593:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c260. 00010000:00000010:2.1:1788201384.334279:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c400. 00010000:00000010:2.1:1788201384.334287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00000080:00000010:2.1:1788201384.581792:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa549f380. 00000080:00000010:2.1:1788201384.581830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd60. 00010000:00000010:2.1:1788201384.588545:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3e00. 00010000:00000010:2.1:1788201384.783284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b903fdf00. 00010000:00000010:2.1:1788201384.783294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba600. 00000080:00000010:2.1:1788201385.088962:0:408129:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc3d280. 00000080:00000010:2.1:1788201385.088979:0:408129:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a80. 00010000:00000010:2.1:1788201385.905138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c31000. 00010000:00000010:2.1:1788201385.922183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952d00. 00010000:00000010:2.1:1788201385.922197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df3e00. 00000080:00000010:2.1:1788201385.931149:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3464a40. 00000080:00000010:2.1:1788201385.931168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c040. 00010000:00000010:2.1:1788201385.945152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6e00. 00010000:00000010:2.1:1788201386.375322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952f00. 00010000:00000010:2.1:1788201386.375330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044c00. 00000080:00000010:2.1:1788201386.382036:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9b8840. 00000080:00000010:2.1:1788201386.382054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8a0. 00010000:00000010:2.1:1788201386.390486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6600. 00010000:00000010:2.1:1788201387.067760:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869eca00. 00010000:00000010:2.1:1788201387.067773:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30400. 00010000:00000010:2.1:1788201387.494629:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec100. 00010000:00000010:2.1:1788201387.494641:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24600. 00010000:00000010:2.1:1788201387.814192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c200. 00010000:00000010:2.1:1788201387.814209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103e00. 00000080:00000010:2.1:1788201388.051538:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbd9f380. 00000080:00000010:2.1:1788201388.051551:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201060. 00010000:00000010:2.1:1788201388.058194:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:2.1:1788201388.854186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039600. 00010000:00000010:2.1:1788201388.854201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00000080:00000010:2.1:1788201390.326256:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3623180. 00000080:00000010:2.1:1788201390.326290:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018c0. 00010000:00000010:2.1:1788201390.335308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030600. 00010000:00000010:2.1:1788201390.355677:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6de00. 00010000:00000010:2.1:1788201390.355693:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7e00. 00000080:00000010:2.1:1788201390.372830:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbde6b40. 00000080:00000010:2.1:1788201390.372843:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468300. 00010000:00000010:2.1:1788201390.380258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030600. 00010000:00000010:2.1:1788201390.444149:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb748d700. 00010000:00000010:2.1:1788201390.444164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6800. 00000080:00000010:2.1:1788201390.449562:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817e5280. 00000080:00000010:2.1:1788201390.449580:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894680c0. 00010000:00000010:2.1:1788201390.995164:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72700. 00010000:00000010:2.1:1788201390.995171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9800. 00010000:00000010:2.1:1788201391.084159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8faced00. 00010000:00000010:2.1:1788201391.084174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8000. 00000080:00000010:2.1:1788201391.103712:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81507380. 00000080:00000010:2.1:1788201391.103723:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:2.1:1788201391.201240:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8face500. 00010000:00000010:2.1:1788201391.201247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044c00. 00000080:00000010:2.1:1788201391.208457:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817b8000. 00000080:00000010:2.1:1788201391.208470:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018a0. 00010000:00000010:2.1:1788201391.215170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00000080:00000010:2.1:1788201392.747142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6a2100. 00000080:00000010:2.1:1788201392.747161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201840. 00000080:00000010:2.1:1788201392.852141:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa348c200. 00000080:00000010:2.1:1788201392.852155:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e40. 00010000:00000010:2.1:1788201392.861142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2e00. 00010000:00000010:2.1:1788201393.854660:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031a00. 00000080:00000010:2.1:1788201394.782646:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85968000. 00000080:00000010:2.1:1788201394.782661:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f60. 00010000:00000010:2.1:1788201394.796062:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:2.1:1788201394.898251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e600. 00010000:00000010:2.1:1788201394.898271:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30800. 00000080:00000010:2.1:1788201394.925443:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf75ac0. 00000080:00000010:2.1:1788201394.925458:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:2.1:1788201394.953026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031a00. 00000080:00000010:2.1:1788201395.080125:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa340eb40. 00000080:00000010:2.1:1788201395.080239:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201ae0. 00010000:00000010:2.1:1788201395.803183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86900. 00010000:00000010:2.1:1788201395.803195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941c600. 00000080:00000010:2.1:1788201395.803202:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa88b9c0. 00000080:00000010:2.1:1788201395.803208:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:2.1:1788201395.837650:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:2.1:1788201395.971178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c200. 00010000:00000010:2.1:1788201395.971194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941de00. 00000080:00000010:2.1:1788201395.980520:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5709080. 00000080:00000010:2.1:1788201395.980537:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd60. 00010000:00000010:2.1:1788201395.985269:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030e00. 00010000:00000010:2.1:1788201395.997176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c000. 00010000:00000010:2.1:1788201395.997184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941ce00. 00000080:00000010:2.1:1788201396.001248:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab54d280. 00000080:00000010:2.1:1788201396.001269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894682c0. 00010000:00000010:2.1:1788201396.273501:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cc00. 00010000:00000010:2.1:1788201396.273515:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653be00. 00010000:00000010:2.1:1788201396.529345:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e100. 00010000:00000010:2.1:1788201396.529371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653b800. 00010000:00000010:2.1:1788201396.890306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ec00. 00010000:00000010:2.1:1788201396.890331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870600. 00010000:00000010:2.1:1788201396.932377:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039e00. 00010000:00000010:2.1:1788201396.932391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201396.996165:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039400. 00010000:00000010:2.1:1788201396.996175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870000. 00000080:00000010:2.1:1788201396.996183:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbef1080. 00000080:00000010:2.1:1788201396.996191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201880. 00010000:00000010:2.1:1788201397.030160:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039100. 00010000:00000010:2.1:1788201397.030172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871a00. 00010000:00000010:2.1:1788201397.665144:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039600. 00010000:00000010:2.1:1788201397.665162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7200. 00010000:00000010:2.1:1788201397.880136:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e000. 00010000:00000010:2.1:1788201397.880150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576400. 00010000:00000010:2.1:1788201399.322261:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9800. 00010000:00000010:2.1:1788201399.342331:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c500. 00010000:00000010:2.1:1788201399.342342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2800. 00000080:00000010:2.1:1788201399.350572:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab658840. 00000080:00000010:2.1:1788201399.350582:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2c0. 00010000:00000010:2.1:1788201399.367123:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06a00. 00010000:00000010:2.1:1788201399.488078:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06e00. 00010000:00000010:2.1:1788201399.515707:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07600. 00010000:00000010:2.1:1788201399.536441:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e500. 00010000:00000010:2.1:1788201399.536457:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00000080:00000010:2.1:1788201399.543820:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa350d280. 00000080:00000010:2.1:1788201399.543890:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca20. 00010000:00000010:2.1:1788201399.583342:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e700. 00010000:00000010:2.1:1788201399.583365:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3400. 00000080:00000010:2.1:1788201399.601245:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8a39c0. 00000080:00000010:2.1:1788201399.601256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468960. 00010000:00000010:2.1:1788201399.605657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06c00. 00010000:00000010:2.1:1788201399.625230:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e300. 00010000:00000010:2.1:1788201399.625242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2c00. 00000080:00000010:2.1:1788201399.634211:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab726300. 00000080:00000010:2.1:1788201399.634229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894689e0. 00010000:00000010:2.1:1788201400.101145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8a00. 00010000:00000010:2.1:1788201400.632292:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d400. 00010000:00000010:2.1:1788201400.632304:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9400. 00000080:00000010:2.1:1788201400.645177:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbce2100. 00000080:00000010:2.1:1788201400.645188:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a20. 00010000:00000010:2.1:1788201401.023131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8400. 00010000:00000010:2.1:1788201401.195665:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ece00. 00010000:00000010:2.1:1788201401.195674:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00000080:00000010:2.1:1788201401.207135:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35c6300. 00000080:00000010:2.1:1788201401.207147:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172340. 00010000:00000010:2.1:1788201401.207155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06800. 00010000:00000010:2.1:1788201401.673646:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b100. 00010000:00000010:2.1:1788201401.673698:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0800. 00000080:00000010:2.1:1788201401.701143:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8141a100. 00000080:00000010:2.1:1788201401.701164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015c0. 00010000:00000010:2.1:1788201401.724150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06c00. 00010000:00000010:2.1:1788201402.598194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bb00. 00010000:00000010:2.1:1788201402.598219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103800. 00000080:00000010:2.1:1788201402.614118:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57f2100. 00000080:00000010:2.1:1788201402.614138:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:2.1:1788201402.634253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07800. 00010000:00000010:2.1:1788201402.672149:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44600. 00010000:00000010:2.1:1788201402.672162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103a00. 00000080:00000010:2.1:1788201402.698287:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7539c0. 00000080:00000010:2.1:1788201402.698307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686a0. 00010000:00000010:2.1:1788201402.714290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07800. 00010000:00000010:2.1:1788201403.004428:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44900. 00010000:00000010:2.1:1788201403.004435:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103400. 00000080:00000010:2.1:1788201403.010887:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3582100. 00000080:00000010:2.1:1788201403.010901:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e00. 00010000:00000010:2.1:1788201403.025936:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07e00. 00010000:00000010:2.1:1788201403.923276:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039b00. 00010000:00000010:2.1:1788201403.923282:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90941c00. 00000080:00000010:2.1:1788201403.929369:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5a7380. 00000080:00000010:2.1:1788201403.929377:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfc0. 00010000:00000010:2.1:1788201403.939503:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07800. 00010000:00000010:2.1:1788201403.969353:0:1353:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039500. 00010000:00000010:2.1:1788201403.969368:0:1353:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90940a00. 00000080:00000010:2.1:1788201403.973487:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c8dac0. 00000080:00000010:2.1:1788201403.973507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468b40. 00010000:00000010:2.1:1788201403.990131:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039700. 00010000:00000010:2.1:1788201403.990138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90941200. 00000080:00000010:2.1:1788201404.055179:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcbe300. 00000080:00000010:2.1:1788201404.055198:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018a0. 00010000:00000010:2.1:1788201404.074689:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039800. 00010000:00000010:2.1:1788201404.074705:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90940c00. 00000080:00000010:2.1:1788201404.087985:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7018c0. 00000080:00000010:2.1:1788201404.088003:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468be0. 00010000:00000010:2.1:1788201404.095727:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06800. 00010000:00000010:2.1:1788201404.655264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b903fdd00. 00010000:00000010:2.1:1788201404.655275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038fc00. 00000080:00000010:2.1:1788201404.659268:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34bf380. 00000080:00000010:2.1:1788201404.659283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f00. 00000080:00000010:2.1:1788201405.059145:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eb8000. 00000080:00000010:2.1:1788201405.059159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b60. 00010000:00000010:2.1:1788201405.348485:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:2.1:1788201405.348498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00000080:00000010:2.1:1788201405.353335:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5cd280. 00000080:00000010:2.1:1788201405.353348:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:2.1:1788201405.358131:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06600. 00010000:00000010:2.1:1788201406.340034:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3cc00. 00010000:00000010:2.1:1788201406.340044:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577200. 00010000:00000010:2.1:1788201406.507301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8c00. 00010000:00000010:2.1:1788201406.861283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9800. 00010000:00000010:2.1:1788201406.866170:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec200. 00010000:00000010:2.1:1788201406.866178:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871c00. 00000080:00000010:2.1:1788201406.871946:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814139c0. 00000080:00000010:2.1:1788201406.871963:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981722e0. 00010000:00000010:2.1:1788201406.881263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07000. 00010000:00000010:2.1:1788201407.187478:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ef00. 00010000:00000010:2.1:1788201407.187484:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871800. 00000080:00000010:2.1:1788201407.192414:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9d39c0. 00000080:00000010:2.1:1788201407.192432:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:2.1:1788201407.195487:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870a00. 00010000:00000010:2.1:1788201407.780315:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86f00. 00010000:00000010:2.1:1788201407.780326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417e00. 00000080:00000010:2.1:1788201407.786493:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3608840. 00000080:00000010:2.1:1788201407.786510:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd00. 00010000:00000010:2.1:1788201407.792227:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871e00. 00000080:00000010:2.1:1788201407.943238:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa370f380. 00000080:00000010:2.1:1788201407.943245:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:2.1:1788201408.344356:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb748d100. 00010000:00000010:2.1:1788201408.344371:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653a800. 00000080:00000010:2.1:1788201408.382172:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8159a940. 00000080:00000010:2.1:1788201408.382190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8a0. 00010000:00000010:2.1:1788201408.387291:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870a00. 00010000:00000010:2.1:1788201408.415506:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb748d200. 00010000:00000010:2.1:1788201408.415514:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa941da00. 00000080:00000010:2.1:1788201408.422484:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbee8000. 00000080:00000010:2.1:1788201408.422501:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468060. 00010000:00000010:2.1:1788201408.427885:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870a00. 00010000:00000010:2.1:1788201409.356861:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952200. 00010000:00000010:2.1:1788201409.356867:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30400. 00010000:00000010:2.1:1788201409.687545:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952300. 00010000:00000010:2.1:1788201409.687552:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06a00. 00000080:00000010:2.1:1788201409.699129:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab624a40. 00000080:00000010:2.1:1788201409.699140:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201880. 00010000:00000010:2.1:1788201409.699147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871400. 00010000:00000010:2.1:1788201410.733182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22100. 00010000:00000010:2.1:1788201410.733195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14400. 00000080:00000010:2.1:1788201410.736453:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcfc200. 00000080:00000010:2.1:1788201410.736466:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:2.1:1788201410.745196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201410.770494:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22600. 00010000:00000010:2.1:1788201410.770504:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14600. 00000080:00000010:2.1:1788201410.774287:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35c1080. 00000080:00000010:2.1:1788201410.774297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468140. 00010000:00000010:2.1:1788201410.779253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201411.481564:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039100. 00010000:00000010:2.1:1788201411.481579:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62200. 00000080:00000010:2.1:1788201411.486327:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcab180. 00000080:00000010:2.1:1788201411.486337:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201740. 00010000:00000010:2.1:1788201411.490358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871a00. 00010000:00000010:2.1:1788201411.508908:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ec00. 00010000:00000010:2.1:1788201411.508916:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b63a00. 00000080:00000010:2.1:1788201411.512719:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5648000. 00000080:00000010:2.1:1788201411.512733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468d60. 00010000:00000010:2.1:1788201411.519847:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871a00. 00010000:00000010:2.1:1788201411.548266:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e200. 00010000:00000010:2.1:1788201411.548295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3b62000. 00000080:00000010:2.1:1788201411.553039:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab589080. 00000080:00000010:2.1:1788201411.553069:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894689e0. 00010000:00000010:2.1:1788201411.670145:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d09200. 00010000:00000010:2.1:1788201411.670164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6c00. 00000080:00000010:2.1:1788201411.677488:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36e8000. 00000080:00000010:2.1:1788201411.677506:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c440. 00010000:00000010:2.1:1788201411.686702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870800. 00010000:00000010:2.1:1788201411.709550:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e300. 00010000:00000010:2.1:1788201411.709557:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7e00. 00000080:00000010:2.1:1788201411.721415:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814d1080. 00000080:00000010:2.1:1788201411.721431:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468fe0. 00010000:00000010:2.1:1788201411.729244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870800. 00010000:00000010:2.1:1788201411.759163:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d09c00. 00010000:00000010:2.1:1788201411.759169:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7000. 00000080:00000010:2.1:1788201411.759177:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5796b40. 00000080:00000010:2.1:1788201411.759182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468560. 00010000:00000010:2.1:1788201411.775452:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870800. 00000080:00000010:2.1:1788201411.844740:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815e2940. 00000080:00000010:2.1:1788201411.844759:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014e0. 00010000:00000010:2.1:1788201411.913177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d09800. 00010000:00000010:2.1:1788201411.913189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6400. 00010000:00000010:2.1:1788201412.073990:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d09100. 00010000:00000010:2.1:1788201412.073997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f8000. 00000080:00000010:2.1:1788201412.087191:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5aa100. 00000080:00000010:2.1:1788201412.087209:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:2.1:1788201412.087218:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7800. 00010000:00000010:2.1:1788201412.713108:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952800. 00010000:00000010:2.1:1788201412.713126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30000. 00010000:00000010:2.1:1788201412.800723:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952f00. 00010000:00000010:2.1:1788201412.800734:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c31a00. 00010000:00000010:2.1:1788201412.856178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952700. 00010000:00000010:2.1:1788201412.856194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30a00. 00000080:00000010:2.1:1788201412.865261:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbff2100. 00000080:00000010:2.1:1788201412.865274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd40. 00010000:00000010:2.1:1788201412.877224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7e00. 00010000:00000010:2.1:1788201412.959880:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039400. 00010000:00000010:2.1:1788201412.959889:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c30800. 00000080:00000010:2.1:1788201412.969517:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa359e300. 00000080:00000010:2.1:1788201412.969532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:2.1:1788201412.978172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6c00. 00010000:00000010:2.1:1788201413.003390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab5039300. 00010000:00000010:2.1:1788201413.003406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3c31c00. 00000080:00000010:2.1:1788201413.011147:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f5d280. 00000080:00000010:2.1:1788201413.011165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468520. 00010000:00000010:2.1:1788201413.019143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6c00. 00000080:00000010:2.1:1788201413.091786:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56d18c0. 00000080:00000010:2.1:1788201413.091801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c20. 00010000:00000010:2.1:1788201413.358413:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8faced00. 00010000:00000010:2.1:1788201413.358428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abb417e00. 00000080:00000010:2.1:1788201413.364235:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5641080. 00000080:00000010:2.1:1788201413.364255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c400. 00010000:00000010:2.1:1788201413.377269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6400. 00000080:00000010:2.1:1788201414.264699:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa354eb40. 00000080:00000010:2.1:1788201414.264716:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f00. 00000080:00000010:2.1:1788201414.376167:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcec200. 00000080:00000010:2.1:1788201414.376204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:2.1:1788201414.598337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9200. 00010000:00000010:2.1:1788201414.615222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c800. 00010000:00000010:2.1:1788201414.615236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6000. 00000080:00000010:2.1:1788201414.624701:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa375ca40. 00000080:00000010:2.1:1788201414.624717:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:2.1:1788201414.657465:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7e00. 00010000:00000010:2.1:1788201414.760472:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86500. 00010000:00000010:2.1:1788201414.760481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c7800. 00000080:00000010:2.1:1788201415.457033:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa343b9c0. 00000080:00000010:2.1:1788201415.457045:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172a80. 00010000:00000010:2.1:1788201415.931229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11d900. 00010000:00000010:2.1:1788201415.931243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038f400. 00010000:00000010:2.1:1788201415.987231:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11de00. 00010000:00000010:2.1:1788201415.987245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038e000. 00010000:00000010:2.1:1788201416.723167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e000. 00010000:00000010:2.1:1788201416.723182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653b600. 00000080:00000010:2.1:1788201416.731315:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8597a100. 00000080:00000010:2.1:1788201416.731334:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018a0. 00010000:00000010:2.1:1788201416.742138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576200. 00010000:00000010:2.1:1788201416.759164:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909ee00. 00010000:00000010:2.1:1788201416.759178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa653ac00. 00000080:00000010:2.1:1788201416.766216:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc218c0. 00000080:00000010:2.1:1788201416.766244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468140. 00010000:00000010:2.1:1788201416.776539:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576200. 00000080:00000010:2.1:1788201417.319557:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab64b180. 00000080:00000010:2.1:1788201417.319571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:2.1:1788201417.423138:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c700. 00010000:00000010:2.1:1788201417.423146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870800. 00010000:00000010:2.1:1788201417.453433:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863ca00. 00010000:00000010:2.1:1788201417.453440:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00000080:00000010:2.1:1788201417.818234:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9cf380. 00000080:00000010:2.1:1788201417.818253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201fe0. 00010000:00000010:2.1:1788201418.676315:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d100. 00010000:00000010:2.1:1788201418.676331:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7e00. 00000080:00000010:2.1:1788201418.708383:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34918c0. 00000080:00000010:2.1:1788201418.708401:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2e0. 00010000:00000010:2.1:1788201418.725145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870800. 00010000:00000010:2.1:1788201419.025601:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6d300. 00010000:00000010:2.1:1788201419.025618:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab30c6400. 00000080:00000010:2.1:1788201419.030635:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3720840. 00000080:00000010:2.1:1788201419.030643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019a0. 00010000:00000010:2.1:1788201419.042125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00000080:00000010:2.1:1788201419.233142:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c27380. 00000080:00000010:2.1:1788201419.233169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00010000:00000010:2.1:1788201419.516186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952700. 00010000:00000010:2.1:1788201419.516200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:2.1:1788201419.527815:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3471080. 00000080:00000010:2.1:1788201419.527824:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201740. 00010000:00000010:2.1:1788201419.540993:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee200. 00010000:00000010:2.1:1788201420.390197:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72100. 00010000:00000010:2.1:1788201420.390215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217a00. 00000080:00000010:2.1:1788201420.398403:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4b0000. 00000080:00000010:2.1:1788201420.398420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc60. 00010000:00000010:2.1:1788201420.414249:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00010000:00000010:2.1:1788201420.437338:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72200. 00010000:00000010:2.1:1788201420.437352:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216800. 00000080:00000010:2.1:1788201420.447217:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3554200. 00000080:00000010:2.1:1788201420.447233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e40. 00010000:00000010:2.1:1788201420.457289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00010000:00000010:2.1:1788201420.572786:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72400. 00010000:00000010:2.1:1788201420.572806:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c216600. 00000080:00000010:2.1:1788201420.583461:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab542940. 00000080:00000010:2.1:1788201420.583503:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:2.1:1788201420.608346:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201420.624385:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6900. 00010000:00000010:2.1:1788201420.624397:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7e00. 00000080:00000010:2.1:1788201420.633145:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8154c200. 00000080:00000010:2.1:1788201420.633175:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468f40. 00010000:00000010:2.1:1788201420.652793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201420.840199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952100. 00010000:00000010:2.1:1788201420.840213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6400. 00010000:00000010:2.1:1788201420.932143:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e300. 00010000:00000010:2.1:1788201420.932160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7000. 00010000:00000010:2.1:1788201420.985172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e700. 00010000:00000010:2.1:1788201420.985185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7400. 00010000:00000010:2.1:1788201421.163146:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4700. 00010000:00000010:2.1:1788201421.163158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577a00. 00010000:00000010:2.1:1788201421.255367:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e500. 00010000:00000010:2.1:1788201421.255380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577800. 00000080:00000010:2.1:1788201421.281265:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc54a40. 00000080:00000010:2.1:1788201421.281398:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:2.1:1788201421.296706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00000080:00000010:2.1:1788201421.561713:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815218c0. 00000080:00000010:2.1:1788201421.561737:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00010000:00000010:2.1:1788201421.607259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef000. 00010000:00000010:2.1:1788201421.811177:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ed00. 00010000:00000010:2.1:1788201421.811193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa038fc00. 00010000:00000010:2.1:1788201422.541394:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b000. 00010000:00000010:2.1:1788201422.541406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef600. 00000080:00000010:2.1:1788201422.969157:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3518840. 00000080:00000010:2.1:1788201422.969167:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:2.1:1788201422.977250:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201422.996976:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44300. 00010000:00000010:2.1:1788201422.996991:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e9400. 00000080:00000010:2.1:1788201423.003189:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7eb180. 00000080:00000010:2.1:1788201423.003201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ec0. 00010000:00000010:2.1:1788201423.012152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201423.025162:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44000. 00010000:00000010:2.1:1788201423.025171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a9f7e8400. 00000080:00000010:2.1:1788201423.031140:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9a6b40. 00000080:00000010:2.1:1788201423.031167:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468760. 00010000:00000010:2.1:1788201423.040572:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201423.241129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef200. 00010000:00000010:2.1:1788201424.263722:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabac00. 00010000:00000010:2.1:1788201424.488268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86f00. 00010000:00000010:2.1:1788201424.488295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870800. 00000080:00000010:2.1:1788201424.491538:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa351a100. 00000080:00000010:2.1:1788201424.491550:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:2.1:1788201424.501681:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee200. 00000080:00000010:2.1:1788201424.567153:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817d3180. 00000080:00000010:2.1:1788201424.567170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468300. 00000080:00000010:2.1:1788201424.662134:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5eeb40. 00000080:00000010:2.1:1788201424.662148:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468940. 00010000:00000010:2.1:1788201424.693168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86900. 00010000:00000010:2.1:1788201424.693174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a90941e00. 00010000:00000010:2.1:1788201424.762645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee200. 00010000:00000010:2.1:1788201425.973296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa6c72a00. 00010000:00000010:2.1:1788201425.973308:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00000080:00000010:2.1:1788201425.987664:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc698c0. 00000080:00000010:2.1:1788201425.987679:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfc0. 00010000:00000010:2.1:1788201425.993132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee600. 00010000:00000010:2.1:1788201426.149521:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c500. 00010000:00000010:2.1:1788201426.149534:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00000080:00000010:2.1:1788201426.156250:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8143b9c0. 00000080:00000010:2.1:1788201426.156338:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201ae0. 00010000:00000010:2.1:1788201426.168418:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201426.213337:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c400. 00010000:00000010:2.1:1788201426.213369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1400. 00000080:00000010:2.1:1788201426.224289:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c62100. 00000080:00000010:2.1:1788201426.224304:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468960. 00010000:00000010:2.1:1788201426.242235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201426.274179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c900. 00010000:00000010:2.1:1788201426.274193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0800. 00010000:00000010:2.1:1788201426.453807:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c800. 00010000:00000010:2.1:1788201426.453823:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0400. 00000080:00000010:2.1:1788201426.486537:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54cd280. 00000080:00000010:2.1:1788201426.486552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201d80. 00010000:00000010:2.1:1788201426.718310:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c500. 00010000:00000010:2.1:1788201426.718322:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:2.1:1788201427.058612:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cd00. 00010000:00000010:2.1:1788201427.058621:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030600. 00010000:00000010:2.1:1788201427.162195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96ba00. 00010000:00000010:2.1:1788201427.162210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2e00. 00010000:00000010:2.1:1788201427.408178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4700. 00010000:00000010:2.1:1788201427.408193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabaa00. 00000080:00000010:2.1:1788201427.423542:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84308000. 00000080:00000010:2.1:1788201427.423558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cac0. 00010000:00000010:2.1:1788201427.435826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef600. 00010000:00000010:2.1:1788201427.458424:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4900. 00010000:00000010:2.1:1788201427.458435:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabbc00. 00000080:00000010:2.1:1788201427.463148:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3498000. 00000080:00000010:2.1:1788201427.463166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ec0. 00010000:00000010:2.1:1788201427.467125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3400. 00010000:00000010:2.1:1788201427.597868:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6db00. 00010000:00000010:2.1:1788201427.597886:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eaba800. 00010000:00000010:2.1:1788201427.999268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a88f6dc00. 00010000:00000010:2.1:1788201427.999285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8eabb600. 00000080:00000010:2.1:1788201428.006586:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf698c0. 00000080:00000010:2.1:1788201428.006605:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172ee0. 00010000:00000010:2.1:1788201428.016228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25800. 00010000:00000010:2.1:1788201428.575624:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b400. 00010000:00000010:2.1:1788201428.575639:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a8c217200. 00000080:00000010:2.1:1788201428.878183:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3595ac0. 00000080:00000010:2.1:1788201428.878195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ccc0. 00000080:00000010:2.1:1788201429.299178:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4fca40. 00000080:00000010:2.1:1788201429.299195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8a0. 00010000:00000010:2.1:1788201429.302394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25a00. 00010000:00000010:2.1:1788201429.419198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ece00. 00010000:00000010:2.1:1788201429.419205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f200. 00000080:00000010:2.1:1788201429.432390:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57e2100. 00000080:00000010:2.1:1788201429.432403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018a0. 00010000:00000010:2.1:1788201429.432412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24000. 00010000:00000010:2.1:1788201429.455736:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec600. 00010000:00000010:2.1:1788201429.455757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6c00. 00000080:00000010:2.1:1788201429.459334:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3509080. 00000080:00000010:2.1:1788201429.459373:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468660. 00010000:00000010:2.1:1788201429.477425:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24000. 00010000:00000010:2.1:1788201430.215165:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec300. 00010000:00000010:2.1:1788201430.215172:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07800. 00000080:00000010:2.1:1788201430.222894:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe62100. 00000080:00000010:2.1:1788201430.222909:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52016e0. 00010000:00000010:2.1:1788201430.255359:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25800. 00000080:00000010:2.1:1788201430.715112:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5438000. 00000080:00000010:2.1:1788201430.715126:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cca0. 00010000:00000010:2.1:1788201430.725246:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:2.1:1788201430.739387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3f00. 00010000:00000010:2.1:1788201430.739403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00000080:00000010:2.1:1788201430.745408:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56c3180. 00000080:00000010:2.1:1788201430.745427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894684c0. 00010000:00000010:2.1:1788201430.756128:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:2.1:1788201430.783614:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3400. 00010000:00000010:2.1:1788201430.783628:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00000080:00000010:2.1:1788201430.788542:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbdf4a40. 00000080:00000010:2.1:1788201430.788559:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468d20. 00010000:00000010:2.1:1788201430.798139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:2.1:1788201430.960399:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3000. 00010000:00000010:2.1:1788201430.960413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00010000:00000010:2.1:1788201431.080412:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3d00. 00010000:00000010:2.1:1788201431.080426:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0600. 00000080:00000010:2.1:1788201431.087194:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf04200. 00000080:00000010:2.1:1788201431.087209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019c0. 00010000:00000010:2.1:1788201431.110144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4200. 00010000:00000010:2.1:1788201431.395009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4e00. 00010000:00000010:2.1:1788201431.411527:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3800. 00010000:00000010:2.1:1788201431.411540:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00000080:00000010:2.1:1788201432.643152:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbd9a940. 00000080:00000010:2.1:1788201432.643172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201bc0. 00010000:00000010:2.1:1788201432.643185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4c00. 00010000:00000010:2.1:1788201433.025159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e600. 00010000:00000010:2.1:1788201433.025166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6000. 00000080:00000010:2.1:1788201433.035144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5710000. 00000080:00000010:2.1:1788201433.035159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201be0. 00010000:00000010:2.1:1788201433.504387:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e100. 00010000:00000010:2.1:1788201433.504399:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24e00. 00010000:00000010:2.1:1788201433.504405:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103e00. 00000080:00000010:2.1:1788201433.516153:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3672100. 00000080:00000010:2.1:1788201433.516172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:2.1:1788201433.527514:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4600. 00010000:00000010:2.1:1788201433.759045:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e500. 00010000:00000010:2.1:1788201433.759051:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24600. 00000080:00000010:2.1:1788201433.769542:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc98000. 00000080:00000010:2.1:1788201433.769554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a20. 00010000:00000010:2.1:1788201433.779137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5400. 00010000:00000010:2.1:1788201434.165828:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b800. 00010000:00000010:2.1:1788201434.165841:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6e00. 00000080:00000010:2.1:1788201434.173143:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e13180. 00000080:00000010:2.1:1788201434.173180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014e0. 00010000:00000010:2.1:1788201434.207967:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4800. 00010000:00000010:2.1:1788201434.694173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6700. 00010000:00000010:2.1:1788201434.694186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5800. 00000080:00000010:2.1:1788201435.254211:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee6300. 00000080:00000010:2.1:1788201435.254238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cbc0. 00010000:00000010:2.1:1788201435.718461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7200. 00010000:00000010:2.1:1788201435.718475:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6df2400. 00010000:00000010:2.1:1788201435.894586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7700. 00010000:00000010:2.1:1788201435.894594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481c00. 00000080:00000010:2.1:1788201435.902137:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37b9080. 00000080:00000010:2.1:1788201435.902147:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a80. 00010000:00000010:2.1:1788201435.910221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3600. 00010000:00000010:2.1:1788201436.246219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24900. 00010000:00000010:2.1:1788201436.246232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480000. 00010000:00000010:2.1:1788201436.455348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00010000:00000010:2.1:1788201436.615480:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952400. 00010000:00000010:2.1:1788201436.615493:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102800. 00000080:00000010:2.1:1788201436.618489:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36c9080. 00000080:00000010:2.1:1788201436.618514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c40. 00010000:00000010:2.1:1788201436.633602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7400. 00010000:00000010:2.1:1788201436.853630:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952d00. 00010000:00000010:2.1:1788201436.853644:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102200. 00010000:00000010:2.1:1788201436.890129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6000. 00010000:00000010:2.1:1788201437.005484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952b00. 00010000:00000010:2.1:1788201437.005492:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102400. 00000080:00000010:2.1:1788201437.015444:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57b3180. 00000080:00000010:2.1:1788201437.021132:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201580. 00010000:00000010:2.1:1788201437.024513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7400. 00010000:00000010:2.1:1788201437.039442:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952c00. 00010000:00000010:2.1:1788201437.039451:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102a00. 00000080:00000010:2.1:1788201437.049163:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e4a940. 00000080:00000010:2.1:1788201437.049204:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894680c0. 00010000:00000010:2.1:1788201437.059566:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7400. 00010000:00000010:2.1:1788201437.356141:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec900. 00010000:00000010:2.1:1788201437.356172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201437.725437:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ea00. 00010000:00000010:2.1:1788201437.725449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9400. 00000080:00000010:2.1:1788201437.770819:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5688000. 00000080:00000010:2.1:1788201437.770851:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:2.1:1788201437.786401:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102400. 00010000:00000010:2.1:1788201438.186237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00010000:00000010:2.1:1788201438.537259:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec600. 00010000:00000010:2.1:1788201438.537277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07e00. 00000080:00000010:2.1:1788201438.541233:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35ab9c0. 00000080:00000010:2.1:1788201438.541261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:2.1:1788201438.550154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102a00. 00010000:00000010:2.1:1788201438.765559:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbe00. 00010000:00000010:2.1:1788201438.765565:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06000. 00010000:00000010:2.1:1788201438.988664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb700. 00010000:00000010:2.1:1788201438.988679:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201438.996899:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee2940. 00000080:00000010:2.1:1788201438.996925:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c140. 00010000:00000010:2.1:1788201439.004908:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102600. 00010000:00000010:2.1:1788201439.314331:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb300. 00010000:00000010:2.1:1788201439.314346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201439.332365:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbd80840. 00000080:00000010:2.1:1788201439.332380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf60. 00010000:00000010:2.1:1788201439.562939:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50000. 00010000:00000010:2.1:1788201439.562948:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5400. 00000080:00000010:2.1:1788201439.566292:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54f4a40. 00000080:00000010:2.1:1788201439.566325:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172f40. 00010000:00000010:2.1:1788201439.581248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102400. 00000080:00000010:2.1:1788201439.917029:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6c7380. 00000080:00000010:2.1:1788201439.917039:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfa0. 00000080:00000010:2.1:1788201439.953263:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f6d280. 00000080:00000010:2.1:1788201439.953420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468820. 00010000:00000010:2.1:1788201440.020905:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50b00. 00010000:00000010:2.1:1788201440.020917:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00010000:00000010:2.1:1788201440.800749:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016a00. 00010000:00000010:2.1:1788201440.800756:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7200. 00000080:00000010:2.1:1788201440.808479:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab610000. 00000080:00000010:2.1:1788201440.808488:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018a0. 00010000:00000010:2.1:1788201440.822203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102e00. 00010000:00000010:2.1:1788201440.868292:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016500. 00010000:00000010:2.1:1788201440.868300:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6600. 00000080:00000010:2.1:1788201440.879379:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814b2940. 00000080:00000010:2.1:1788201440.879395:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd00. 00010000:00000010:2.1:1788201441.247395:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eea00. 00010000:00000010:2.1:1788201441.327265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016f00. 00010000:00000010:2.1:1788201441.327277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5102000. 00000080:00000010:2.1:1788201441.339211:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8596ca40. 00000080:00000010:2.1:1788201441.339233:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c80. 00010000:00000010:2.1:1788201441.345175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15800. 00010000:00000010:2.1:1788201441.412254:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016200. 00010000:00000010:2.1:1788201441.412259:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103c00. 00000080:00000010:2.1:1788201441.417202:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8594f380. 00000080:00000010:2.1:1788201441.417220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c260. 00010000:00000010:2.1:1788201441.431194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15600. 00010000:00000010:2.1:1788201441.448758:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016400. 00010000:00000010:2.1:1788201441.448771:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103800. 00000080:00000010:2.1:1788201441.462194:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf2c200. 00000080:00000010:2.1:1788201441.462212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468cc0. 00010000:00000010:2.1:1788201441.472055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14c00. 00000080:00000010:2.1:1788201441.899426:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab78b180. 00000080:00000010:2.1:1788201441.899442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201700. 00010000:00000010:2.1:1788201442.117218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e800. 00010000:00000010:2.1:1788201442.117230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1400. 00010000:00000010:2.1:1788201442.556185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14600. 00010000:00000010:2.1:1788201442.588344:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7a00. 00010000:00000010:2.1:1788201442.588358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25000. 00000080:00000010:2.1:1788201442.607593:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37a18c0. 00000080:00000010:2.1:1788201442.607608:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e00. 00010000:00000010:2.1:1788201442.615209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14600. 00010000:00000010:2.1:1788201442.657232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7500. 00010000:00000010:2.1:1788201442.657251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24e00. 00000080:00000010:2.1:1788201442.669229:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81451080. 00000080:00000010:2.1:1788201442.669247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686a0. 00010000:00000010:2.1:1788201442.683887:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14600. 00010000:00000010:2.1:1788201443.717333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14e00. 00000080:00000010:2.1:1788201444.544144:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa2f380. 00000080:00000010:2.1:1788201444.544165:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ccc0. 00010000:00000010:2.1:1788201444.550337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5000. 00010000:00000010:2.1:1788201444.606243:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86500. 00010000:00000010:2.1:1788201444.606256:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576e00. 00000080:00000010:2.1:1788201444.615229:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814d2940. 00000080:00000010:2.1:1788201444.615239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201fe0. 00010000:00000010:2.1:1788201444.627183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5800. 00010000:00000010:2.1:1788201444.653150:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86c00. 00010000:00000010:2.1:1788201444.653156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576600. 00000080:00000010:2.1:1788201444.657362:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84808840. 00000080:00000010:2.1:1788201444.657405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468960. 00000080:00000010:2.1:1788201445.625151:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5b3180. 00000080:00000010:2.1:1788201445.625176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c940. 00010000:00000010:2.1:1788201445.707278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb900. 00010000:00000010:2.1:1788201445.707291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149ca00. 00010000:00000010:2.1:1788201446.562202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3500. 00010000:00000010:2.1:1788201446.562213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201446.567160:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3476300. 00000080:00000010:2.1:1788201446.567199:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:2.1:1788201446.573144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00010000:00000010:2.1:1788201446.601062:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84dc3d00. 00010000:00000010:2.1:1788201446.601070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:2.1:1788201446.605238:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbff98c0. 00000080:00000010:2.1:1788201446.605267:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468860. 00010000:00000010:2.1:1788201446.622133:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00000080:00000010:2.1:1788201447.916149:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9ea940. 00000080:00000010:2.1:1788201447.916192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:2.1:1788201448.450128:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4200. 00010000:00000010:2.1:1788201448.639417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba124ab00. 00010000:00000010:2.1:1788201448.639428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15c00. 00010000:00000010:2.1:1788201449.116245:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8fc50500. 00010000:00000010:2.1:1788201449.116251:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0800. 00000080:00000010:2.1:1788201449.121160:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c11080. 00000080:00000010:2.1:1788201449.121185:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e20. 00010000:00000010:2.1:1788201449.145140:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5000. 00000080:00000010:2.1:1788201449.600576:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa363ca40. 00000080:00000010:2.1:1788201449.600599:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc40. 00000080:00000010:2.1:1788201449.749173:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5422940. 00000080:00000010:2.1:1788201449.749187:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c440. 00010000:00000010:2.1:1788201449.762299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5800. 00010000:00000010:2.1:1788201449.803428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00010000:00000010:2.1:1788201449.813255:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b900. 00010000:00000010:2.1:1788201449.813263:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab5103000. 00000080:00000010:2.1:1788201449.834155:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf0e300. 00000080:00000010:2.1:1788201449.834179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c220. 00010000:00000010:2.1:1788201449.834189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5800. 00000080:00000010:2.1:1788201450.314168:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ed0840. 00000080:00000010:2.1:1788201450.314197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f20. 00010000:00000010:2.1:1788201450.324258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00010000:00000010:2.1:1788201450.346515:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:2.1:1788201450.346524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7abf2f9800. 00000080:00000010:2.1:1788201450.353695:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c818c0. 00000080:00000010:2.1:1788201450.353724:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468760. 00010000:00000010:2.1:1788201450.370076:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00000080:00000010:2.1:1788201450.948549:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815a98c0. 00000080:00000010:2.1:1788201450.948571:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cae0. 00010000:00000010:2.1:1788201450.954340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4e00. 00000080:00000010:2.1:1788201451.012786:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa348a940. 00000080:00000010:2.1:1788201451.012820:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468080. 00010000:00000010:2.1:1788201451.038134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4800. 00010000:00000010:2.1:1788201451.075451:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6e00. 00010000:00000010:2.1:1788201451.075459:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82800. 00000080:00000010:2.1:1788201451.084857:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8cc200. 00000080:00000010:2.1:1788201451.089504:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468320. 00010000:00000010:2.1:1788201451.113137:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4800. 00010000:00000010:2.1:1788201451.261170:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6600. 00010000:00000010:2.1:1788201451.261178:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eec00. 00010000:00000010:2.1:1788201451.500617:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24200. 00010000:00000010:2.1:1788201451.500634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef000. 00000080:00000010:2.1:1788201451.505376:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85905280. 00000080:00000010:2.1:1788201451.505415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf60. 00010000:00000010:2.1:1788201451.517287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5200. 00010000:00000010:2.1:1788201451.549763:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98d24f00. 00010000:00000010:2.1:1788201451.549781:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00000080:00000010:2.1:1788201451.562590:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbd83180. 00000080:00000010:2.1:1788201451.562625:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468be0. 00010000:00000010:2.1:1788201451.572304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5200. 00010000:00000010:2.1:1788201452.190237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4000. 00010000:00000010:2.1:1788201454.594622:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e400. 00010000:00000010:2.1:1788201454.594630:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25600. 00000080:00000010:2.1:1788201454.601609:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84133180. 00000080:00000010:2.1:1788201454.601646:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2e0. 00010000:00000010:2.1:1788201454.607448:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5000. 00010000:00000010:2.1:1788201454.687183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb600. 00010000:00000010:2.1:1788201454.687198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25400. 00000080:00000010:2.1:1788201454.708204:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5633180. 00000080:00000010:2.1:1788201454.708218:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2e0. 00010000:00000010:2.1:1788201455.201401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbb00. 00010000:00000010:2.1:1788201455.201414:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480000. 00010000:00000010:2.1:1788201456.923610:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44700. 00010000:00000010:2.1:1788201456.923625:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1000. 00010000:00000010:2.1:1788201457.043966:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44300. 00010000:00000010:2.1:1788201457.043979:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0e00. 00000080:00000010:2.1:1788201457.051394:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5b9080. 00000080:00000010:2.1:1788201457.051532:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c80. 00010000:00000010:2.1:1788201457.418268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737df00. 00010000:00000010:2.1:1788201457.418275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045800. 00000080:00000010:2.1:1788201457.426755:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84b62100. 00000080:00000010:2.1:1788201457.426775:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:2.1:1788201457.438663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:2.1:1788201457.453418:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d500. 00010000:00000010:2.1:1788201457.453432:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045600. 00000080:00000010:2.1:1788201457.470168:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57098c0. 00000080:00000010:2.1:1788201457.470190:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894686c0. 00010000:00000010:2.1:1788201457.487294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:2.1:1788201457.500529:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d900. 00010000:00000010:2.1:1788201457.500537:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045c00. 00000080:00000010:2.1:1788201457.506253:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa896b40. 00000080:00000010:2.1:1788201457.506280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894687c0. 00010000:00000010:2.1:1788201457.532817:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00000080:00000010:2.1:1788201457.756793:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36edac0. 00000080:00000010:2.1:1788201457.756843:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f00. 00010000:00000010:2.1:1788201458.164318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:2.1:1788201458.164330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00000080:00000010:2.1:1788201458.171200:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e54200. 00000080:00000010:2.1:1788201458.171231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b60. 00000080:00000010:0.1:1788201459.432519:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9118c0. 00000080:00000010:0.1:1788201459.432548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce80. 00010000:00000010:0.1:1788201459.440649:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201459.465233:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b000. 00010000:00000010:0.1:1788201459.465238:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90395e00. 00000080:00000010:0.1:1788201459.472832:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c74a40. 00000080:00000010:0.1:1788201459.472847:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2c0. 00010000:00000010:0.1:1788201459.477368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201459.501183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbd00. 00010000:00000010:0.1:1788201459.501215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90395c00. 00000080:00000010:0.1:1788201459.512214:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eb6b40. 00000080:00000010:0.1:1788201459.512278:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce80. 00010000:00000010:0.1:1788201459.527150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201459.573750:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb500. 00010000:00000010:0.1:1788201459.573757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90395800. 00000080:00000010:0.1:1788201459.579411:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35d2100. 00000080:00000010:0.1:1788201459.579428:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:0.1:1788201459.585285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201459.604215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbe00. 00010000:00000010:0.1:1788201459.604239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90394800. 00000080:00000010:0.1:1788201459.627305:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf11080. 00000080:00000010:0.1:1788201459.627342:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce80. 00010000:00000010:0.1:1788201459.632709:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201459.694251:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016b00. 00010000:00000010:0.1:1788201459.694265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90395000. 00000080:00000010:0.1:1788201459.706330:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84b25ac0. 00000080:00000010:0.1:1788201459.706344:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015e0. 00010000:00000010:0.1:1788201459.716159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201459.788351:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016d00. 00010000:00000010:0.1:1788201459.788361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00000080:00000010:0.1:1788201459.795326:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3496300. 00000080:00000010:0.1:1788201459.795341:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014e0. 00010000:00000010:0.1:1788201459.801152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201459.865012:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016800. 00010000:00000010:0.1:1788201459.865026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647000. 00000080:00000010:0.1:1788201459.872178:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85828840. 00000080:00000010:0.1:1788201459.872235:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201740. 00010000:00000010:0.1:1788201459.886167:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201459.898331:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016500. 00010000:00000010:0.1:1788201459.898346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646200. 00000080:00000010:0.1:1788201459.919440:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84beb180. 00000080:00000010:0.1:1788201459.919465:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:0.1:1788201459.984288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7900. 00010000:00000010:0.1:1788201459.984304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646a00. 00000080:00000010:0.1:1788201459.994626:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81566b40. 00000080:00000010:0.1:1788201459.994643:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c00. 00010000:00000010:0.1:1788201460.001915:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201460.019578:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016200. 00010000:00000010:0.1:1788201460.019586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646000. 00000080:00000010:0.1:1788201460.026433:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cae300. 00000080:00000010:0.1:1788201460.026459:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:0.1:1788201460.031254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201460.053724:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016c00. 00010000:00000010:0.1:1788201460.053732:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647400. 00000080:00000010:0.1:1788201460.053751:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab56d280. 00000080:00000010:0.1:1788201460.053767:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:0.1:1788201460.059987:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201460.144961:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016e00. 00010000:00000010:0.1:1788201460.144973:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646c00. 00000080:00000010:0.1:1788201460.151979:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbecc200. 00000080:00000010:0.1:1788201460.151991:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981720a0. 00010000:00000010:0.1:1788201460.161372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201460.179261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016900. 00010000:00000010:0.1:1788201460.179267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646e00. 00000080:00000010:0.1:1788201460.183257:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56fe300. 00000080:00000010:0.1:1788201460.183313:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:0.1:1788201460.195217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201460.214879:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016000. 00010000:00000010:0.1:1788201460.214890:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646600. 00000080:00000010:0.1:1788201460.226890:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf5b9c0. 00000080:00000010:0.1:1788201460.226909:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:0.1:1788201460.356180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7600. 00010000:00000010:0.1:1788201460.356202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87696600. 00000080:00000010:0.1:1788201460.365300:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab492100. 00000080:00000010:0.1:1788201460.365312:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:0.1:1788201460.376272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201460.397172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7800. 00010000:00000010:0.1:1788201460.397197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697e00. 00000080:00000010:0.1:1788201460.408055:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ca2100. 00000080:00000010:0.1:1788201460.408104:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:0.1:1788201460.423142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00000080:00000010:0.1:1788201460.509156:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5998c0. 00000080:00000010:0.1:1788201460.509195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:0.1:1788201460.517123:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:0.1:1788201460.536328:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952400. 00010000:00000010:0.1:1788201460.536342:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87696800. 00000080:00000010:0.1:1788201460.554130:0:408146:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7e18c0. 00000080:00000010:0.1:1788201460.554156:0:408146:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:0.1:1788201460.568258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00010000:00000010:2.1:1788201460.595312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44900. 00010000:00000010:2.1:1788201460.595326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5200. 00000080:00000010:0.1:1788201460.644144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab572100. 00000080:00000010:0.1:1788201460.644178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce20. 00010000:00000010:0.1:1788201460.644187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201460.681943:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44800. 00010000:00000010:0.1:1788201460.681958:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00000080:00000010:0.1:1788201460.697322:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf58840. 00000080:00000010:0.1:1788201460.697362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:0.1:1788201460.702384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:2.1:1788201460.899813:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44200. 00010000:00000010:2.1:1788201460.899821:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5e00. 00000080:00000010:2.1:1788201460.907116:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4ae300. 00000080:00000010:2.1:1788201460.907130:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f40. 00010000:00000010:2.1:1788201460.919652:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:0.1:1788201460.994364:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44000. 00010000:00000010:0.1:1788201460.994372:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4200. 00000080:00000010:0.1:1788201461.003144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34498c0. 00000080:00000010:0.1:1788201461.003159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201640. 00010000:00000010:0.1:1788201461.010307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201461.025524:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84500. 00010000:00000010:0.1:1788201461.025534:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5600. 00000080:00000010:0.1:1788201461.030357:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81485ac0. 00000080:00000010:0.1:1788201461.030380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:0.1:1788201461.038552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201461.064268:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84800. 00010000:00000010:0.1:1788201461.064282:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4e00. 00000080:00000010:0.1:1788201461.067341:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7239c0. 00000080:00000010:0.1:1788201461.067360:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cbc0. 00010000:00000010:0.1:1788201461.076251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:2.1:1788201461.099372:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84200. 00010000:00000010:2.1:1788201461.099384:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00000080:00000010:2.1:1788201461.099391:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5716b40. 00000080:00000010:2.1:1788201461.099398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:2.1:1788201461.108134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:0.1:1788201461.201182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84000. 00010000:00000010:0.1:1788201461.201205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82000. 00000080:00000010:0.1:1788201461.207366:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3526300. 00000080:00000010:0.1:1788201461.207389:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201400. 00010000:00000010:0.1:1788201461.224226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201461.418302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84e00. 00010000:00000010:0.1:1788201461.418312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00000080:00000010:0.1:1788201461.425449:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe0ca40. 00000080:00000010:0.1:1788201461.425462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f80. 00010000:00000010:0.1:1788201461.433178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201461.495208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84100. 00010000:00000010:0.1:1788201461.495216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00000080:00000010:2.1:1788201461.498467:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8430b9c0. 00000080:00000010:2.1:1788201461.498503:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201200. 00010000:00000010:0.1:1788201461.558147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4000. 00010000:00000010:0.1:1788201461.587158:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84b00. 00010000:00000010:0.1:1788201461.587166:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83c00. 00000080:00000010:0.1:1788201461.592154:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf44200. 00000080:00000010:0.1:1788201461.592172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ee0. 00010000:00000010:0.1:1788201461.603474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:2.1:1788201461.628192:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb300. 00010000:00000010:2.1:1788201461.628205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83e00. 00010000:00000010:0.1:1788201461.730395:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44b00. 00010000:00000010:0.1:1788201461.730402:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82e00. 00000080:00000010:0.1:1788201461.742100:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa368a100. 00000080:00000010:0.1:1788201461.742119:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201880. 00010000:00000010:0.1:1788201461.749255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201461.784167:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44c00. 00010000:00000010:0.1:1788201461.784178:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82c00. 00000080:00000010:0.1:1788201461.784198:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab674a40. 00000080:00000010:0.1:1788201461.784206:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cbc0. 00010000:00000010:0.1:1788201461.793265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201461.815456:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c000. 00010000:00000010:0.1:1788201461.815464:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83a00. 00000080:00000010:0.1:1788201461.827202:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84b56b40. 00000080:00000010:0.1:1788201461.827234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd40. 00010000:00000010:0.1:1788201461.839281:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201461.995170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb000. 00010000:00000010:0.1:1788201461.995193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182400. 00000080:00000010:0.1:1788201462.002229:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3567380. 00000080:00000010:0.1:1788201462.002247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201540. 00010000:00000010:0.1:1788201462.023417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201462.050105:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb800. 00010000:00000010:0.1:1788201462.050115:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183800. 00000080:00000010:0.1:1788201462.055215:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35ddac0. 00000080:00000010:0.1:1788201462.055239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd40. 00010000:00000010:0.1:1788201462.063140:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201462.211291:0:14:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d300. 00010000:00000010:0.1:1788201462.211298:0:14:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183200. 00000080:00000010:0.1:1788201462.219246:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7c8840. 00000080:00000010:0.1:1788201462.219293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201cc0. 00010000:00000010:0.1:1788201462.223409:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00000080:00000010:2.1:1788201462.254883:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf04a40. 00000080:00000010:2.1:1788201462.254902:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd40. 00010000:00000010:2.1:1788201462.259039:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5400. 00010000:00000010:2.1:1788201462.285031:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbb00. 00010000:00000010:2.1:1788201462.285040:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182000. 00000080:00000010:2.1:1788201462.289238:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8406b9c0. 00000080:00000010:2.1:1788201462.289279:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468860. 00010000:00000010:2.1:1788201462.289289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5400. 00010000:00000010:0.1:1788201462.478365:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ef00. 00010000:00000010:0.1:1788201462.478372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182e00. 00000080:00000010:0.1:1788201462.478382:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa63180. 00000080:00000010:0.1:1788201462.478423:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981723a0. 00010000:00000010:0.1:1788201462.478430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201462.571209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e600. 00010000:00000010:0.1:1788201462.571223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0200. 00000080:00000010:0.1:1788201462.583191:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57c5280. 00000080:00000010:0.1:1788201462.583223:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e20. 00010000:00000010:0.1:1788201462.592126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201462.615980:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e800. 00010000:00000010:0.1:1788201462.615988:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0400. 00000080:00000010:0.1:1788201462.627548:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5414200. 00000080:00000010:0.1:1788201462.627575:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:0.1:1788201462.636162:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:2.1:1788201462.648271:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ea00. 00010000:00000010:2.1:1788201462.648287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00000080:00000010:0.1:1788201462.662178:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3577380. 00000080:00000010:0.1:1788201462.662201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c400. 00010000:00000010:0.1:1788201462.670333:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201462.728489:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11eb00. 00010000:00000010:0.1:1788201462.728507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0e00. 00000080:00000010:0.1:1788201462.739313:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9e6300. 00000080:00000010:0.1:1788201462.739337:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201200. 00010000:00000010:0.1:1788201462.750538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201462.775322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737dc00. 00010000:00000010:0.1:1788201462.775334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00000080:00000010:0.1:1788201462.808530:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5339c0. 00000080:00000010:0.1:1788201462.808548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00000080:00000010:0.1:1788201462.872307:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab632940. 00000080:00000010:0.1:1788201462.877143:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:0.1:1788201462.892135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201462.958152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201463.098288:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e200. 00010000:00000010:0.1:1788201463.098307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1400. 00000080:00000010:0.1:1788201463.118166:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57018c0. 00000080:00000010:0.1:1788201463.118202:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f80. 00010000:00000010:0.1:1788201463.153127:0:14:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4200. 00010000:00000010:0.1:1788201463.218813:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e300. 00010000:00000010:0.1:1788201463.218827:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1a00. 00000080:00000010:2.1:1788201463.240217:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc6ca40. 00000080:00000010:2.1:1788201463.240253:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201400. 00010000:00000010:0.1:1788201463.265742:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4800. 00010000:00000010:0.1:1788201463.296176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e700. 00010000:00000010:0.1:1788201463.296190:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0600. 00000080:00000010:0.1:1788201463.306811:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3654a40. 00000080:00000010:0.1:1788201463.306850:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e00. 00010000:00000010:0.1:1788201463.315233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201463.379293:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7700. 00010000:00000010:0.1:1788201463.379300:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:0.1:1788201463.394319:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfe2100. 00000080:00000010:0.1:1788201463.394350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201640. 00010000:00000010:0.1:1788201463.394360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:2.1:1788201463.443122:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d700. 00010000:00000010:2.1:1788201463.443128:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e200. 00000080:00000010:2.1:1788201463.444506:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfe2940. 00000080:00000010:2.1:1788201463.444529:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:2.1:1788201463.459100:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5400. 00010000:00000010:0.1:1788201463.524084:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44e00. 00010000:00000010:0.1:1788201463.524111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ee00. 00000080:00000010:0.1:1788201463.529700:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc2c200. 00000080:00000010:0.1:1788201463.529749:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468960. 00010000:00000010:0.1:1788201463.547373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00000080:00000010:0.1:1788201463.760137:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcf2100. 00000080:00000010:0.1:1788201463.760173:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:0.1:1788201463.765141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00000080:00000010:0.1:1788201463.838321:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6518c0. 00000080:00000010:0.1:1788201463.838364:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f40. 00010000:00000010:0.1:1788201463.841139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201463.903168:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201463.903173:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:0.1:1788201463.908159:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab530840. 00000080:00000010:0.1:1788201463.908176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:0.1:1788201463.926202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201463.949183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c100. 00010000:00000010:0.1:1788201463.949194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:0.1:1788201463.979872:0:13:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81558000. 00000080:00000010:0.1:1788201463.979912:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00010000:00000010:0.1:1788201463.979924:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:2.1:1788201463.991245:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cd00. 00010000:00000010:2.1:1788201463.991251:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:0.1:1788201464.001428:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc04a40. 00000080:00000010:0.1:1788201464.001449:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfa0. 00010000:00000010:0.1:1788201464.001458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00010000:00000010:0.1:1788201464.027300:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:0.1:1788201464.027311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:0.1:1788201464.040481:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f718c0. 00000080:00000010:0.1:1788201464.040515:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00010000:00000010:0.1:1788201464.044489:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.073882:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863ce00. 00010000:00000010:0.1:1788201464.073892:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:0.1:1788201464.086446:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3502100. 00000080:00000010:0.1:1788201464.086481:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfa0. 00010000:00000010:0.1:1788201464.114222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.149261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cc00. 00010000:00000010:0.1:1788201464.149477:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fa00. 00000080:00000010:0.1:1788201464.149488:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbef9080. 00000080:00000010:0.1:1788201464.149496:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cea0. 00010000:00000010:0.1:1788201464.153839:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.371160:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c200. 00010000:00000010:0.1:1788201464.371167:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7800. 00000080:00000010:0.1:1788201464.381181:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf0ca40. 00000080:00000010:0.1:1788201464.381217:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f60. 00010000:00000010:0.1:1788201464.388226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:2.1:1788201464.396154:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bc00. 00010000:00000010:2.1:1788201464.396171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6800. 00000080:00000010:0.1:1788201464.408475:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8587ca40. 00000080:00000010:0.1:1788201464.408498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cea0. 00010000:00000010:0.1:1788201464.416309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.482126:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4000. 00010000:00000010:0.1:1788201464.512186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7200. 00010000:00000010:0.1:1788201464.512200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6600. 00000080:00000010:0.1:1788201464.512212:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8e8000. 00000080:00000010:0.1:1788201464.512239:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b00. 00010000:00000010:2.1:1788201464.604166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7d00. 00010000:00000010:2.1:1788201464.604179:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3000. 00000080:00000010:2.1:1788201464.614201:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:2.1:1788201464.622528:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4e00. 00010000:00000010:2.1:1788201464.636302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7400. 00010000:00000010:2.1:1788201464.636310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2e00. 00000080:00000010:2.1:1788201464.644641:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa347a100. 00000080:00000010:2.1:1788201464.644697:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e40. 00010000:00000010:2.1:1788201464.649357:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4e00. 00010000:00000010:2.1:1788201464.684182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7100. 00010000:00000010:2.1:1788201464.684195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2c00. 00000080:00000010:0.1:1788201464.684246:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37918c0. 00000080:00000010:0.1:1788201464.684256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894681e0. 00010000:00000010:0.1:1788201464.696438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.714414:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7f00. 00010000:00000010:0.1:1788201464.714421:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3400. 00000080:00000010:0.1:1788201464.736017:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e2a940. 00000080:00000010:0.1:1788201464.736048:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cea0. 00010000:00000010:0.1:1788201464.736055:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.816450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016700. 00010000:00000010:0.1:1788201464.816462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045e00. 00000080:00000010:0.1:1788201464.823146:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7b2940. 00000080:00000010:0.1:1788201464.823161:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201740. 00010000:00000010:0.1:1788201464.833235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.846186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016300. 00010000:00000010:0.1:1788201464.846199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045800. 00000080:00000010:0.1:1788201464.865387:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37a3180. 00000080:00000010:0.1:1788201464.865422:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cea0. 00010000:00000010:0.1:1788201464.865433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.884590:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016000. 00010000:00000010:0.1:1788201464.884597:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045600. 00000080:00000010:0.1:1788201464.891700:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3414200. 00000080:00000010:0.1:1788201464.891733:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:0.1:1788201464.906142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.920193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b400. 00010000:00000010:0.1:1788201464.920207:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647800. 00000080:00000010:0.1:1788201464.924378:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b843cd280. 00000080:00000010:0.1:1788201464.924416:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:0.1:1788201464.929932:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:0.1:1788201464.947512:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b100. 00010000:00000010:0.1:1788201464.947519:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647c00. 00000080:00000010:0.1:1788201464.960395:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c22940. 00000080:00000010:0.1:1788201464.960420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cea0. 00010000:00000010:0.1:1788201464.964550:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647c00. 00010000:00000010:0.1:1788201464.989416:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b700. 00010000:00000010:0.1:1788201464.989430:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647e00. 00000080:00000010:0.1:1788201464.993406:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81530000. 00000080:00000010:0.1:1788201464.993419:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:0.1:1788201464.999175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4600. 00010000:00000010:0.1:1788201465.061633:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b600. 00010000:00000010:0.1:1788201465.061645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646e00. 00000080:00000010:0.1:1788201465.066253:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6998c0. 00000080:00000010:0.1:1788201465.066263:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201d80. 00010000:00000010:0.1:1788201465.134553:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016e00. 00010000:00000010:0.1:1788201465.134565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647a00. 00000080:00000010:0.1:1788201465.144162:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab722100. 00000080:00000010:0.1:1788201465.144194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201ae0. 00010000:00000010:0.1:1788201465.159107:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646e00. 00010000:00000010:0.1:1788201465.168181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016600. 00010000:00000010:0.1:1788201465.168188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646400. 00000080:00000010:0.1:1788201465.177592:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab50f380. 00000080:00000010:0.1:1788201465.177613:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc60. 00010000:00000010:2.1:1788201465.301228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016400. 00010000:00000010:2.1:1788201465.301235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647200. 00000080:00000010:2.1:1788201465.309244:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4ff380. 00000080:00000010:2.1:1788201465.309283:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00010000:00000010:2.1:1788201465.313211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00010000:00000010:2.1:1788201465.331458:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016500. 00010000:00000010:2.1:1788201465.331465:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646200. 00000080:00000010:2.1:1788201465.337380:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f46300. 00000080:00000010:2.1:1788201465.337410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468940. 00010000:00000010:2.1:1788201465.344264:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00010000:00000010:2.1:1788201465.371828:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016800. 00010000:00000010:2.1:1788201465.371835:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647000. 00000080:00000010:2.1:1788201465.376213:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbeca940. 00000080:00000010:2.1:1788201465.376242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e40. 00010000:00000010:2.1:1788201465.384389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00010000:00000010:0.1:1788201465.400307:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b200. 00010000:00000010:0.1:1788201465.400320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00010000:00000010:2.1:1788201465.673906:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b300. 00010000:00000010:2.1:1788201465.673918:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00010000:00000010:2.1:1788201465.749188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bb00. 00010000:00000010:2.1:1788201465.749203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2800. 00000080:00000010:2.1:1788201465.764863:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34adac0. 00000080:00000010:2.1:1788201465.764881:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201be0. 00010000:00000010:0.1:1788201465.781170:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5400. 00010000:00000010:0.1:1788201465.800553:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86900. 00010000:00000010:0.1:1788201465.800564:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2200. 00000080:00000010:0.1:1788201465.809619:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc939c0. 00000080:00000010:0.1:1788201465.822176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468b60. 00010000:00000010:0.1:1788201465.833151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00010000:00000010:0.1:1788201465.862423:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86a00. 00010000:00000010:0.1:1788201465.862435:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3800. 00010000:00000010:0.1:1788201465.933191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016d00. 00010000:00000010:0.1:1788201465.933201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13600. 00000080:00000010:0.1:1788201465.954581:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab64d280. 00000080:00000010:0.1:1788201465.954600:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019a0. 00010000:00000010:0.1:1788201465.964155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00010000:00000010:0.1:1788201465.983188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016100. 00010000:00000010:0.1:1788201465.983211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13000. 00000080:00000010:0.1:1788201465.996603:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81540000. 00000080:00000010:0.1:1788201465.996623:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c760. 00010000:00000010:0.1:1788201466.023157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00010000:00000010:0.1:1788201466.142336:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbe00. 00010000:00000010:0.1:1788201466.142343:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f12a00. 00000080:00000010:0.1:1788201466.155766:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84824200. 00000080:00000010:0.1:1788201466.155784:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201840. 00010000:00000010:0.1:1788201466.155793:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00010000:00000010:2.1:1788201466.272144:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb800. 00010000:00000010:2.1:1788201466.272162:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f12400. 00000080:00000010:2.1:1788201466.302551:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f9c200. 00000080:00000010:2.1:1788201466.302586:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201780. 00010000:00000010:2.1:1788201466.337198:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5400. 00010000:00000010:0.1:1788201466.387422:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb000. 00010000:00000010:0.1:1788201466.387438:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13a00. 00000080:00000010:0.1:1788201466.400163:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf35ac0. 00000080:00000010:0.1:1788201466.400187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ee0. 00010000:00000010:0.1:1788201466.408458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00000080:00000010:2.1:1788201466.518150:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e698c0. 00000080:00000010:2.1:1788201466.518166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201fe0. 00010000:00000010:2.1:1788201466.538148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:2.1:1788201466.598234:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb700. 00010000:00000010:2.1:1788201466.598251:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f12600. 00000080:00000010:2.1:1788201466.607230:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81582940. 00000080:00000010:2.1:1788201466.607260:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468b40. 00000080:00000010:2.1:1788201466.635193:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9d9080. 00000080:00000010:2.1:1788201466.635206:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894681a0. 00010000:00000010:0.1:1788201466.639327:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4e00. 00010000:00000010:0.1:1788201466.654252:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb900. 00010000:00000010:0.1:1788201466.654265:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13c00. 00000080:00000010:0.1:1788201466.667223:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eeca40. 00000080:00000010:0.1:1788201466.667255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468080. 00010000:00000010:0.1:1788201466.693175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00010000:00000010:0.1:1788201466.797228:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb300. 00010000:00000010:0.1:1788201466.797236:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13400. 00000080:00000010:0.1:1788201466.797245:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7ae300. 00000080:00000010:0.1:1788201466.797252:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b80. 00010000:00000010:0.1:1788201466.808209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00010000:00000010:0.1:1788201466.864138:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c900. 00010000:00000010:0.1:1788201466.864146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0600. 00000080:00000010:0.1:1788201466.870234:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa368eb40. 00000080:00000010:0.1:1788201466.870251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201760. 00010000:00000010:0.1:1788201466.872236:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00000080:00000010:2.1:1788201466.897729:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f02100. 00000080:00000010:2.1:1788201466.901034:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c040. 00010000:00000010:2.1:1788201466.905137:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:2.1:1788201466.915306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c000. 00010000:00000010:2.1:1788201466.915320:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1c00. 00000080:00000010:2.1:1788201466.933163:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab590000. 00000080:00000010:2.1:1788201466.933189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e00. 00010000:00000010:2.1:1788201466.933199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:0.1:1788201467.021496:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84100. 00010000:00000010:0.1:1788201467.021507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1000. 00000080:00000010:0.1:1788201467.026144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9d4200. 00000080:00000010:0.1:1788201467.026158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e40. 00010000:00000010:0.1:1788201467.044506:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00010000:00000010:0.1:1788201467.090035:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84c00. 00010000:00000010:0.1:1788201467.090047:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:0.1:1788201467.168180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84d00. 00010000:00000010:0.1:1788201467.168211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0e00. 00000080:00000010:0.1:1788201467.181204:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e818c0. 00000080:00000010:0.1:1788201467.181219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201100. 00010000:00000010:0.1:1788201467.263989:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84400. 00010000:00000010:0.1:1788201467.264004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00000080:00000010:0.1:1788201467.270483:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37bca40. 00000080:00000010:0.1:1788201467.270495:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201480. 00010000:00000010:0.1:1788201467.279159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:2.1:1788201467.452289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84200. 00010000:00000010:2.1:1788201467.452309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182e00. 00000080:00000010:0.1:1788201467.460145:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6a0840. 00000080:00000010:0.1:1788201467.460163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52016e0. 00010000:00000010:0.1:1788201467.472137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201467.494182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84800. 00010000:00000010:0.1:1788201467.494194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182600. 00000080:00000010:0.1:1788201467.504141:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56dd280. 00000080:00000010:0.1:1788201467.504167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca60. 00010000:00000010:0.1:1788201467.511303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201467.536249:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84500. 00010000:00000010:0.1:1788201467.536263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183e00. 00000080:00000010:0.1:1788201467.561825:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ff8840. 00000080:00000010:0.1:1788201467.561844:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2e0. 00010000:00000010:0.1:1788201467.565258:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201467.579313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b000. 00010000:00000010:0.1:1788201467.579322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183a00. 00000080:00000010:0.1:1788201467.585411:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc898c0. 00000080:00000010:0.1:1788201467.585438:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca60. 00010000:00000010:0.1:1788201467.591739:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201467.619292:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b300. 00010000:00000010:0.1:1788201467.619299:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183400. 00000080:00000010:0.1:1788201467.623422:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6718c0. 00000080:00000010:0.1:1788201467.623441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cca0. 00010000:00000010:0.1:1788201467.628126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201467.641173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b600. 00010000:00000010:0.1:1788201467.641189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182000. 00000080:00000010:0.1:1788201467.658153:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa098c0. 00000080:00000010:0.1:1788201467.658170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2e0. 00010000:00000010:0.1:1788201467.663147:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201467.674495:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b900. 00010000:00000010:0.1:1788201467.674510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182a00. 00000080:00000010:0.1:1788201467.680199:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54b4a40. 00000080:00000010:0.1:1788201467.684454:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca60. 00010000:00000010:0.1:1788201467.694741:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201467.709236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7700. 00010000:00000010:0.1:1788201467.709245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183200. 00000080:00000010:0.1:1788201467.716215:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab69e300. 00000080:00000010:0.1:1788201467.716241:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cca0. 00010000:00000010:0.1:1788201467.726152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00000080:00000010:0.1:1788201467.971550:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa567ca40. 00000080:00000010:0.1:1788201467.971581:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52017a0. 00010000:00000010:0.1:1788201467.980284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201468.021114:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86300. 00010000:00000010:0.1:1788201468.021130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183c00. 00000080:00000010:0.1:1788201468.030418:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85963180. 00000080:00000010:0.1:1788201468.030430:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca60. 00010000:00000010:0.1:1788201468.041173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201468.172603:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86f00. 00010000:00000010:0.1:1788201468.172614:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00000080:00000010:0.1:1788201468.182226:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814c0000. 00000080:00000010:0.1:1788201468.182251:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019c0. 00010000:00000010:0.1:1788201468.188157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201468.223387:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86d00. 00010000:00000010:0.1:1788201468.223398:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030600. 00000080:00000010:0.1:1788201468.240666:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa90d280. 00000080:00000010:0.1:1788201468.240702:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca60. 00010000:00000010:0.1:1788201468.250438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00010000:00000010:2.1:1788201468.421168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7500. 00010000:00000010:2.1:1788201468.421180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030800. 00000080:00000010:0.1:1788201468.441748:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8146f380. 00000080:00000010:0.1:1788201468.441770:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c80. 00010000:00000010:0.1:1788201468.455137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83000. 00010000:00000010:0.1:1788201468.554186:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e400. 00010000:00000010:0.1:1788201468.554200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00000080:00000010:0.1:1788201468.577173:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f618c0. 00000080:00000010:0.1:1788201468.577186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c80. 00010000:00000010:0.1:1788201468.636283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ec00. 00010000:00000010:0.1:1788201468.636296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00000080:00000010:0.1:1788201468.650399:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3772940. 00000080:00000010:0.1:1788201468.650441:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201900. 00010000:00000010:0.1:1788201468.650452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00010000:00000010:0.1:1788201468.784144:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ed00. 00010000:00000010:0.1:1788201468.784160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6800. 00000080:00000010:0.1:1788201468.794350:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf94200. 00000080:00000010:0.1:1788201468.794432:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018c0. 00010000:00000010:0.1:1788201468.800134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00010000:00000010:0.1:1788201468.888146:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e800. 00010000:00000010:0.1:1788201468.888167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6400. 00000080:00000010:0.1:1788201468.904159:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5dca940. 00000080:00000010:0.1:1788201468.904195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019c0. 00010000:00000010:0.1:1788201468.911391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00010000:00000010:0.1:1788201468.994253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00010000:00000010:0.1:1788201469.013178:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7500. 00010000:00000010:0.1:1788201469.013186:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6600. 00000080:00000010:0.1:1788201469.019705:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab492940. 00000080:00000010:0.1:1788201469.019744:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201700. 00010000:00000010:0.1:1788201469.032141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00010000:00000010:0.1:1788201469.257623:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7400. 00010000:00000010:0.1:1788201469.257638:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7800. 00000080:00000010:0.1:1788201469.282133:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5eca40. 00000080:00000010:0.1:1788201469.282168:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52016e0. 00010000:00000010:0.1:1788201469.322415:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7f00. 00010000:00000010:0.1:1788201469.322431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6a00. 00000080:00000010:0.1:1788201469.340461:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbed6b40. 00000080:00000010:0.1:1788201469.340507:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc60. 00010000:00000010:0.1:1788201469.349268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201469.383284:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7200. 00010000:00000010:0.1:1788201469.383296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7e00. 00000080:00000010:0.1:1788201469.398222:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8151eb40. 00000080:00000010:0.1:1788201469.398240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd60. 00010000:00000010:0.1:1788201469.423196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5400. 00010000:00000010:0.1:1788201469.491135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045c00. 00010000:00000010:0.1:1788201469.556257:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86a00. 00010000:00000010:0.1:1788201469.556285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045600. 00000080:00000010:0.1:1788201469.563221:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8481d280. 00000080:00000010:0.1:1788201469.563261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201da0. 00010000:00000010:0.1:1788201469.574216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201469.605871:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86900. 00010000:00000010:0.1:1788201469.605878:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045800. 00010000:00000010:2.1:1788201469.675436:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016100. 00010000:00000010:2.1:1788201469.675451:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044800. 00000080:00000010:0.1:1788201469.690218:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3449080. 00000080:00000010:0.1:1788201469.690237:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172820. 00010000:00000010:0.1:1788201469.705141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201469.722263:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016d00. 00010000:00000010:0.1:1788201469.722277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481e00. 00000080:00000010:0.1:1788201469.736712:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe1a100. 00000080:00000010:0.1:1788201469.736753:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:0.1:1788201469.741875:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201469.845339:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016500. 00010000:00000010:0.1:1788201469.845346:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480800. 00000080:00000010:0.1:1788201469.861140:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa548eb40. 00000080:00000010:0.1:1788201469.861183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f00. 00010000:00000010:0.1:1788201469.861193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201469.879211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e600. 00010000:00000010:0.1:1788201469.879228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481600. 00000080:00000010:0.1:1788201469.889606:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa72100. 00000080:00000010:0.1:1788201469.889642:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:0.1:1788201469.897257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201469.985788:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016400. 00010000:00000010:0.1:1788201469.985802:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481200. 00000080:00000010:0.1:1788201469.988438:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5722100. 00000080:00000010:0.1:1788201469.988453:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201760. 00010000:00000010:0.1:1788201469.996139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.020308:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016200. 00010000:00000010:0.1:1788201470.020323:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480a00. 00000080:00000010:0.1:1788201470.031235:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa345eb40. 00000080:00000010:0.1:1788201470.031269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:0.1:1788201470.042253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201470.091161:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e500. 00010000:00000010:2.1:1788201470.091168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7481400. 00000080:00000010:0.1:1788201470.091262:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6c5280. 00000080:00000010:0.1:1788201470.091388:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e40. 00010000:00000010:0.1:1788201470.103165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.120177:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e000. 00010000:00000010:0.1:1788201470.120188:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00000080:00000010:0.1:1788201470.128549:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84b66300. 00000080:00000010:0.1:1788201470.128574:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:0.1:1788201470.137142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.159072:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb100. 00010000:00000010:0.1:1788201470.159080:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646400. 00000080:00000010:0.1:1788201470.165264:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817d5ac0. 00000080:00000010:0.1:1788201470.165298:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00010000:00000010:0.1:1788201470.176721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.202194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbd00. 00010000:00000010:0.1:1788201470.202205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646e00. 00000080:00000010:2.1:1788201470.208572:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa359a940. 00000080:00000010:2.1:1788201470.208588:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c9e0. 00010000:00000010:0.1:1788201470.212130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00010000:00000010:0.1:1788201470.227185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb000. 00010000:00000010:0.1:1788201470.227199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647a00. 00000080:00000010:0.1:1788201470.245373:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9da940. 00000080:00000010:0.1:1788201470.245405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468100. 00010000:00000010:0.1:1788201470.249431:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201470.265390:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb800. 00010000:00000010:2.1:1788201470.265398:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647c00. 00000080:00000010:2.1:1788201470.271610:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cb98c0. 00000080:00000010:2.1:1788201470.271624:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:2.1:1788201470.275138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4a00. 00010000:00000010:2.1:1788201470.293180:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb500. 00010000:00000010:2.1:1788201470.293188:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00000080:00000010:2.1:1788201470.300251:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81593180. 00000080:00000010:2.1:1788201470.300264:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468d60. 00010000:00000010:2.1:1788201470.306138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4a00. 00010000:00000010:0.1:1788201470.331313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016c00. 00010000:00000010:0.1:1788201470.331327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647800. 00010000:00000010:2.1:1788201470.355149:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016f00. 00010000:00000010:2.1:1788201470.355165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647000. 00000080:00000010:0.1:1788201470.358241:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f62100. 00000080:00000010:0.1:1788201470.358332:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981723a0. 00010000:00000010:0.1:1788201470.372702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.393306:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016600. 00010000:00000010:0.1:1788201470.393313:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646200. 00000080:00000010:0.1:1788201470.393320:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7a1080. 00000080:00000010:0.1:1788201470.393341:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf60. 00010000:00000010:0.1:1788201470.399206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.418305:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016e00. 00010000:00000010:0.1:1788201470.418311:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647200. 00000080:00000010:0.1:1788201470.424154:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35c2940. 00000080:00000010:0.1:1788201470.424192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:0.1:1788201470.436154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.455401:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016900. 00010000:00000010:0.1:1788201470.455413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646a00. 00000080:00000010:0.1:1788201470.460148:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc97380. 00000080:00000010:0.1:1788201470.460176:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c3a0. 00010000:00000010:0.1:1788201470.465147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.489463:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016000. 00010000:00000010:0.1:1788201470.489474:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646000. 00000080:00000010:0.1:1788201470.498329:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3428840. 00000080:00000010:0.1:1788201470.498340:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:0.1:1788201470.506533:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201470.525179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016300. 00010000:00000010:2.1:1788201470.525191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647400. 00000080:00000010:0.1:1788201470.535348:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54af380. 00000080:00000010:0.1:1788201470.542966:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf60. 00010000:00000010:0.1:1788201470.548143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.609181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c200. 00010000:00000010:0.1:1788201470.609197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647e00. 00000080:00000010:0.1:1788201470.628269:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe61080. 00000080:00000010:0.1:1788201470.628299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b80. 00010000:00000010:0.1:1788201470.645608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.743296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.760258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863ca00. 00010000:00000010:0.1:1788201470.760270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13200. 00000080:00000010:0.1:1788201470.767362:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8178a100. 00000080:00000010:0.1:1788201470.767405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201840. 00010000:00000010:0.1:1788201470.777211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.872172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cc00. 00010000:00000010:0.1:1788201470.872181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13600. 00000080:00000010:0.1:1788201470.872189:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab58b9c0. 00000080:00000010:0.1:1788201470.872195:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172800. 00010000:00000010:0.1:1788201470.877663:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.899429:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863ce00. 00010000:00000010:0.1:1788201470.899436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f200. 00000080:00000010:0.1:1788201470.899446:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7f6b40. 00000080:00000010:0.1:1788201470.899467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2a0. 00010000:00000010:0.1:1788201470.909919:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.919283:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:0.1:1788201470.919296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fa00. 00000080:00000010:0.1:1788201470.926584:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85954a40. 00000080:00000010:0.1:1788201470.926615:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cac0. 00010000:00000010:0.1:1788201470.936307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201470.955262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cd00. 00010000:00000010:0.1:1788201470.955284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:0.1:1788201470.964134:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ce6b40. 00000080:00000010:0.1:1788201470.964164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cae0. 00010000:00000010:0.1:1788201470.968137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201471.113147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201471.157753:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7300. 00010000:00000010:0.1:1788201471.157760:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:0.1:1788201471.224454:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7100. 00010000:00000010:0.1:1788201471.224464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:0.1:1788201471.229150:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36fdac0. 00000080:00000010:0.1:1788201471.229169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201bc0. 00010000:00000010:0.1:1788201471.240748:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201471.349304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6900. 00010000:00000010:0.1:1788201471.349318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:0.1:1788201471.357346:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab625ac0. 00000080:00000010:0.1:1788201471.357386:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014a0. 00010000:00000010:0.1:1788201471.374248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201471.483138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201471.498281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7d00. 00010000:00000010:0.1:1788201471.498300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:0.1:1788201471.505527:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbdce300. 00000080:00000010:0.1:1788201471.505542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a20. 00010000:00000010:0.1:1788201471.520257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00000080:00000010:0.1:1788201471.548229:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b843c6b40. 00000080:00000010:0.1:1788201471.548265:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8c0. 00010000:00000010:0.1:1788201471.551222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201471.573309:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7600. 00010000:00000010:0.1:1788201471.573317:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82c00. 00000080:00000010:0.1:1788201471.577451:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4ab9c0. 00000080:00000010:0.1:1788201471.577488:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2c0. 00010000:00000010:0.1:1788201471.588406:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201471.676756:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6400. 00010000:00000010:0.1:1788201471.676770:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82600. 00000080:00000010:0.1:1788201471.688525:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbcca940. 00000080:00000010:0.1:1788201471.688543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201820. 00010000:00000010:0.1:1788201471.720450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201471.793459:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201471.793474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83c00. 00000080:00000010:0.1:1788201471.825171:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85960000. 00000080:00000010:0.1:1788201471.825209:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201ae0. 00010000:00000010:0.1:1788201471.841784:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201472.007840:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6600. 00010000:00000010:0.1:1788201472.007846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00000080:00000010:0.1:1788201472.114214:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37ea940. 00000080:00000010:0.1:1788201472.114252:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981720a0. 00010000:00000010:0.1:1788201472.126379:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201472.165193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b829c6f00. 00010000:00000010:0.1:1788201472.165209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83400. 00000080:00000010:0.1:1788201472.176248:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa0eb40. 00000080:00000010:0.1:1788201472.176269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c400. 00010000:00000010:0.1:1788201472.189463:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201472.337245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7400. 00010000:00000010:0.1:1788201472.337255:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82a00. 00000080:00000010:0.1:1788201472.360141:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5daca40. 00000080:00000010:0.1:1788201472.360189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201740. 00010000:00000010:0.1:1788201472.416350:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7000. 00010000:00000010:0.1:1788201472.416361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82000. 00000080:00000010:0.1:1788201472.436255:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa378c200. 00000080:00000010:0.1:1788201472.436294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c220. 00010000:00000010:0.1:1788201472.557148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201472.576595:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737db00. 00010000:00000010:0.1:1788201472.576609:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031400. 00000080:00000010:0.1:1788201472.591063:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eea940. 00000080:00000010:0.1:1788201472.591086:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:0.1:1788201472.603307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201472.729298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737de00. 00010000:00000010:0.1:1788201472.729311:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201472.913124:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d300. 00010000:00000010:0.1:1788201472.913132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030e00. 00010000:00000010:0.1:1788201472.963425:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7300. 00010000:00000010:0.1:1788201472.963434:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031a00. 00000080:00000010:0.1:1788201472.988146:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab7fc200. 00000080:00000010:0.1:1788201472.988180:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b00. 00010000:00000010:2.1:1788201473.056221:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7500. 00010000:00000010:2.1:1788201473.056233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00000080:00000010:2.1:1788201473.056247:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf4b180. 00000080:00000010:2.1:1788201473.056284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc40. 00010000:00000010:2.1:1788201473.075310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00010000:00000010:0.1:1788201473.183374:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7900. 00010000:00000010:0.1:1788201473.183381:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25400. 00000080:00000010:0.1:1788201473.189195:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe54a40. 00000080:00000010:0.1:1788201473.189234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201680. 00010000:00000010:0.1:1788201473.193126:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201473.221150:0:1032:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7600. 00010000:00000010:0.1:1788201473.221156:0:1032:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25000. 00000080:00000010:0.1:1788201473.224227:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8146eb40. 00000080:00000010:0.1:1788201473.224244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c440. 00010000:00000010:0.1:1788201473.270673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201473.290417:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d500. 00010000:00000010:0.1:1788201473.290438:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25600. 00010000:00000010:0.1:1788201473.383442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e800. 00010000:00000010:0.1:1788201473.383453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24600. 00000080:00000010:0.1:1788201473.391873:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa368f380. 00000080:00000010:0.1:1788201473.391910:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:0.1:1788201473.398772:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201473.431166:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ea00. 00010000:00000010:2.1:1788201473.431172:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25800. 00000080:00000010:2.1:1788201473.436253:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81452100. 00000080:00000010:2.1:1788201473.436314:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c440. 00010000:00000010:2.1:1788201473.443228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00010000:00000010:2.1:1788201473.492316:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11ed00. 00010000:00000010:2.1:1788201473.492333:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24a00. 00000080:00000010:2.1:1788201473.509815:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa365b9c0. 00000080:00000010:2.1:1788201473.509847:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468b60. 00010000:00000010:2.1:1788201473.520134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00010000:00000010:0.1:1788201473.550236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11eb00. 00010000:00000010:0.1:1788201473.550248:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25e00. 00000080:00000010:0.1:1788201473.576019:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8ba940. 00000080:00000010:0.1:1788201473.576055:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ec0. 00010000:00000010:0.1:1788201473.585895:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201473.616715:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e100. 00010000:00000010:2.1:1788201473.616726:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24400. 00000080:00000010:2.1:1788201473.629246:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa992940. 00000080:00000010:2.1:1788201473.629284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf20. 00010000:00000010:2.1:1788201473.629294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0400. 00000080:00000010:2.1:1788201473.699951:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cb2100. 00000080:00000010:2.1:1788201473.699969:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468760. 00010000:00000010:2.1:1788201473.713076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0600. 00010000:00000010:0.1:1788201473.732681:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e200. 00010000:00000010:0.1:1788201473.732689:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24c00. 00000080:00000010:0.1:1788201473.743181:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c8ca40. 00000080:00000010:0.1:1788201473.743209:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ca0. 00010000:00000010:0.1:1788201473.750650:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201473.784232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e400. 00010000:00000010:0.1:1788201473.784247:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24200. 00000080:00000010:0.1:1788201473.784258:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81566300. 00000080:00000010:0.1:1788201473.784266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:0.1:1788201473.792312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201473.831206:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bad11e300. 00010000:00000010:0.1:1788201473.831213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24000. 00000080:00000010:0.1:1788201473.838049:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5798840. 00000080:00000010:0.1:1788201473.838060:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf20. 00010000:00000010:0.1:1788201473.852268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201473.949340:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7800. 00010000:00000010:0.1:1788201473.949353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5e00. 00000080:00000010:0.1:1788201473.962207:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab795280. 00000080:00000010:0.1:1788201473.962247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f40. 00010000:00000010:0.1:1788201473.972499:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201474.017199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7b00. 00010000:00000010:0.1:1788201474.017214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4400. 00010000:00000010:0.1:1788201474.082185:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7700. 00010000:00000010:0.1:1788201474.082200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4a00. 00000080:00000010:0.1:1788201474.106257:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37b39c0. 00000080:00000010:0.1:1788201474.106292:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:0.1:1788201474.111126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201474.154311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b900. 00010000:00000010:2.1:1788201474.154327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5400. 00010000:00000010:2.1:1788201474.228528:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b600. 00010000:00000010:2.1:1788201474.228535:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5a00. 00000080:00000010:2.1:1788201474.238144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b98a17380. 00000080:00000010:2.1:1788201474.238179:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172b60. 00010000:00000010:2.1:1788201474.250001:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00010000:00000010:2.1:1788201474.370323:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b000. 00010000:00000010:2.1:1788201474.370339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb7480e00. 00000080:00000010:2.1:1788201474.407259:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab703180. 00000080:00000010:2.1:1788201474.407287:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201400. 00010000:00000010:2.1:1788201474.433144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1400. 00010000:00000010:0.1:1788201474.580194:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b300. 00010000:00000010:0.1:1788201474.580200:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7000. 00000080:00000010:0.1:1788201474.597237:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35eeb40. 00000080:00000010:0.1:1788201474.597272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201580. 00010000:00000010:0.1:1788201474.615699:0:1032:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:0.1:1788201474.698169:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bf00. 00010000:00000010:0.1:1788201474.698182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7200. 00000080:00000010:0.1:1788201474.717443:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf6d280. 00000080:00000010:0.1:1788201474.717463:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f80. 00010000:00000010:2.1:1788201474.724688:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00000080:00000010:0.1:1788201474.875496:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85883180. 00000080:00000010:0.1:1788201474.875524:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201d40. 00010000:00000010:0.1:1788201474.879337:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00010000:00000010:0.1:1788201475.161211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bb00. 00010000:00000010:0.1:1788201475.161221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00000080:00000010:0.1:1788201475.184128:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eb8840. 00000080:00000010:0.1:1788201475.184154:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e20. 00010000:00000010:0.1:1788201475.194079:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00010000:00000010:2.1:1788201475.363199:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bc00. 00010000:00000010:2.1:1788201475.363214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697600. 00000080:00000010:2.1:1788201475.376152:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815439c0. 00000080:00000010:2.1:1788201475.376190:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c40. 00010000:00000010:2.1:1788201475.390147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00010000:00000010:2.1:1788201475.431281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b000. 00010000:00000010:2.1:1788201475.431293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87696200. 00000080:00000010:2.1:1788201475.435185:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5651080. 00000080:00000010:2.1:1788201475.435212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468520. 00010000:00000010:2.1:1788201475.440137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00010000:00000010:2.1:1788201475.506281:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86600. 00010000:00000010:2.1:1788201475.506288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4600. 00000080:00000010:2.1:1788201475.510983:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa8d4200. 00000080:00000010:2.1:1788201475.511009:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201cc0. 00010000:00000010:2.1:1788201475.515193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00010000:00000010:0.1:1788201475.536217:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86d00. 00010000:00000010:0.1:1788201475.536238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5800. 00000080:00000010:0.1:1788201475.541535:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81529080. 00000080:00000010:0.1:1788201475.541554:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ee0. 00010000:00000010:0.1:1788201475.825665:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec600. 00010000:00000010:0.1:1788201475.825686:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697200. 00000080:00000010:0.1:1788201475.842564:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab5aa940. 00000080:00000010:0.1:1788201475.842600:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201cc0. 00010000:00000010:0.1:1788201475.847355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:2.1:1788201475.882296:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec800. 00010000:00000010:2.1:1788201475.882314:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87696a00. 00000080:00000010:0.1:1788201475.900216:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfa8000. 00000080:00000010:0.1:1788201475.900249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:0.1:1788201475.916081:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201475.964176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec300. 00010000:00000010:0.1:1788201475.964188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87696000. 00000080:00000010:2.1:1788201475.994141:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3703180. 00000080:00000010:2.1:1788201475.994152:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2c0. 00010000:00000010:2.1:1788201476.005128:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6600. 00010000:00000010:2.1:1788201476.058358:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecf00. 00010000:00000010:2.1:1788201476.058368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697400. 00000080:00000010:0.1:1788201476.067141:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa375dac0. 00000080:00000010:0.1:1788201476.067157:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468a40. 00010000:00000010:0.1:1788201476.067166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.129241:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecb00. 00010000:00000010:0.1:1788201476.129259:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697a00. 00000080:00000010:0.1:1788201476.140186:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa89b9c0. 00000080:00000010:0.1:1788201476.140214:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201540. 00010000:00000010:0.1:1788201476.146172:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.167312:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecc00. 00010000:00000010:0.1:1788201476.167324:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87696c00. 00000080:00000010:0.1:1788201476.167332:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9318c0. 00000080:00000010:0.1:1788201476.167357:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00000080:00000010:0.1:1788201476.233120:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa579b180. 00000080:00000010:0.1:1788201476.233135:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e20. 00010000:00000010:0.1:1788201476.238204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.255207:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec900. 00010000:00000010:0.1:1788201476.255232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87696800. 00000080:00000010:0.1:1788201476.267060:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa567e300. 00000080:00000010:0.1:1788201476.267078:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:0.1:1788201476.273151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.354212:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952000. 00010000:00000010:0.1:1788201476.354219:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697c00. 00000080:00000010:0.1:1788201476.359384:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84b7e300. 00000080:00000010:0.1:1788201476.359413:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201d40. 00010000:00000010:0.1:1788201476.366233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.388821:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952100. 00010000:00000010:0.1:1788201476.388834:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697e00. 00000080:00000010:0.1:1788201476.399774:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85886b40. 00000080:00000010:0.1:1788201476.399790:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:0.1:1788201476.412746:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.475276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952b00. 00010000:00000010:0.1:1788201476.475286:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6200. 00000080:00000010:0.1:1788201476.486080:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa374b9c0. 00000080:00000010:0.1:1788201476.486147:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201200. 00010000:00000010:0.1:1788201476.507369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.529625:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952e00. 00010000:00000010:0.1:1788201476.529633:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7000. 00000080:00000010:0.1:1788201476.534391:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81768000. 00000080:00000010:0.1:1788201476.534430:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:0.1:1788201476.545161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.577027:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952f00. 00010000:00000010:0.1:1788201476.577042:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4400. 00000080:00000010:0.1:1788201476.578455:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fa3180. 00000080:00000010:0.1:1788201476.578498:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c140. 00010000:00000010:0.1:1788201476.586142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.609734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952700. 00010000:00000010:0.1:1788201476.609744:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5e00. 00000080:00000010:0.1:1788201476.618261:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3588000. 00000080:00000010:0.1:1788201476.618307:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:0.1:1788201476.622156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:2.1:1788201476.641161:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952500. 00010000:00000010:2.1:1788201476.641168:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4600. 00000080:00000010:2.1:1788201476.649302:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34fb9c0. 00000080:00000010:2.1:1788201476.649316:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c140. 00010000:00000010:2.1:1788201476.655295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06a00. 00010000:00000010:2.1:1788201476.688730:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952d00. 00010000:00000010:2.1:1788201476.688737:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4800. 00010000:00000010:0.1:1788201476.699608:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7200. 00010000:00000010:0.1:1788201476.758264:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952600. 00010000:00000010:0.1:1788201476.758371:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5a00. 00000080:00000010:0.1:1788201476.777664:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab768840. 00000080:00000010:0.1:1788201476.777683:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201400. 00010000:00000010:0.1:1788201476.781283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.795189:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952900. 00010000:00000010:0.1:1788201476.795201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5c00. 00000080:00000010:0.1:1788201476.805190:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fe5ac0. 00000080:00000010:0.1:1788201476.805203:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:0.1:1788201476.813132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.834229:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952300. 00010000:00000010:0.1:1788201476.834236:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4a00. 00000080:00000010:0.1:1788201476.840388:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3556300. 00000080:00000010:0.1:1788201476.840427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cac0. 00010000:00000010:0.1:1788201476.840437:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.861161:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952800. 00010000:00000010:0.1:1788201476.861167:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4000. 00000080:00000010:0.1:1788201476.865228:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35798c0. 00000080:00000010:0.1:1788201476.865247:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2a0. 00010000:00000010:0.1:1788201476.878223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201476.897211:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952400. 00010000:00000010:0.1:1788201476.897218:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5000. 00000080:00000010:0.1:1788201476.911025:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37b98c0. 00000080:00000010:0.1:1788201476.911074:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:0.1:1788201476.921136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201477.047264:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016300. 00010000:00000010:0.1:1788201477.047371:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e4200. 00000080:00000010:0.1:1788201477.055712:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34d8840. 00000080:00000010:0.1:1788201477.055745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201000. 00010000:00000010:0.1:1788201477.063369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201477.078280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec100. 00010000:00000010:0.1:1788201477.078294:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5200. 00010000:00000010:0.1:1788201477.130181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ece00. 00010000:00000010:0.1:1788201477.130195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb75e5800. 00000080:00000010:0.1:1788201477.138303:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817c5ac0. 00000080:00000010:0.1:1788201477.138322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201640. 00010000:00000010:0.1:1788201477.152583:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201477.174425:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb600. 00010000:00000010:0.1:1788201477.174433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:0.1:1788201477.215450:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbe00. 00010000:00000010:0.1:1788201477.215467:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3800. 00000080:00000010:0.1:1788201477.219162:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81422940. 00000080:00000010:0.1:1788201477.219220:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f20. 00010000:00000010:0.1:1788201477.231198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201477.361253:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb000. 00010000:00000010:0.1:1788201477.361269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3e00. 00000080:00000010:0.1:1788201477.389501:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbff2940. 00000080:00000010:0.1:1788201477.389543:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201aa0. 00010000:00000010:0.1:1788201477.389552:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:2.1:1788201477.435489:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbd00. 00010000:00000010:2.1:1788201477.435503:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00000080:00000010:2.1:1788201477.456730:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37339c0. 00000080:00000010:2.1:1788201477.456771:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca20. 00010000:00000010:2.1:1788201477.460233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07e00. 00010000:00000010:0.1:1788201477.481188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb100. 00010000:00000010:0.1:1788201477.481202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2a00. 00000080:00000010:0.1:1788201477.493197:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cf0000. 00000080:00000010:0.1:1788201477.493213:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894680a0. 00010000:00000010:0.1:1788201477.499214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201477.528233:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb300. 00010000:00000010:0.1:1788201477.528241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3c00. 00000080:00000010:0.1:1788201477.537220:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56af380. 00000080:00000010:0.1:1788201477.537261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:0.1:1788201477.544144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:2.1:1788201477.578188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbbf00. 00010000:00000010:2.1:1788201477.578198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2400. 00010000:00000010:2.1:1788201477.616984:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbba00. 00010000:00000010:2.1:1788201477.616991:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3600. 00000080:00000010:2.1:1788201477.623748:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56dca40. 00000080:00000010:2.1:1788201477.623779:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172200. 00010000:00000010:2.1:1788201477.633812:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07c00. 00010000:00000010:2.1:1788201477.664262:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb900. 00010000:00000010:2.1:1788201477.664279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00000080:00000010:0.1:1788201477.673040:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbde3180. 00000080:00000010:0.1:1788201477.673074:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468820. 00010000:00000010:0.1:1788201477.684260:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201477.781203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84fbb700. 00010000:00000010:0.1:1788201477.781219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2c00. 00000080:00000010:0.1:1788201477.804147:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54e5ac0. 00000080:00000010:0.1:1788201477.804255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201260. 00010000:00000010:0.1:1788201477.815380:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201477.849191:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bb00. 00010000:00000010:0.1:1788201477.849208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2e00. 00000080:00000010:0.1:1788201477.856884:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5fd8840. 00000080:00000010:0.1:1788201477.856895:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c9e0. 00010000:00000010:0.1:1788201477.863141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201477.999353:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016000. 00010000:00000010:0.1:1788201477.999367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577c00. 00000080:00000010:0.1:1788201478.021299:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3753180. 00000080:00000010:0.1:1788201478.021312:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f60. 00010000:00000010:0.1:1788201478.025946:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.041183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016900. 00010000:00000010:0.1:1788201478.041199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577600. 00000080:00000010:0.1:1788201478.072834:0:13:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85978840. 00000080:00000010:0.1:1788201478.077181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfa0. 00010000:00000010:0.1:1788201478.095280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016e00. 00010000:00000010:0.1:1788201478.095293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577a00. 00000080:00000010:0.1:1788201478.109260:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8166ca40. 00000080:00000010:0.1:1788201478.109295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:0.1:1788201478.115301:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07000. 00010000:00000010:0.1:1788201478.207154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.227210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016c00. 00010000:00000010:0.1:1788201478.227224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576000. 00000080:00000010:0.1:1788201478.234783:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815bd280. 00000080:00000010:0.1:1788201478.234817:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201740. 00010000:00000010:0.1:1788201478.247132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.266377:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016200. 00010000:00000010:0.1:1788201478.266391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576c00. 00000080:00000010:0.1:1788201478.276150:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab633180. 00000080:00000010:0.1:1788201478.276177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:0.1:1788201478.285526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.304963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016400. 00010000:00000010:0.1:1788201478.304971:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576800. 00000080:00000010:0.1:1788201478.313525:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3526b40. 00000080:00000010:0.1:1788201478.313553:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c940. 00010000:00000010:0.1:1788201478.318146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.337146:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016500. 00010000:00000010:0.1:1788201478.337154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577e00. 00000080:00000010:0.1:1788201478.341801:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85882100. 00000080:00000010:0.1:1788201478.341829:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:0.1:1788201478.352251:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.435170:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016d00. 00010000:00000010:0.1:1788201478.435187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576a00. 00000080:00000010:0.1:1788201478.442865:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815a4a40. 00000080:00000010:0.1:1788201478.442971:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015e0. 00010000:00000010:0.1:1788201478.454377:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.482484:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96be00. 00010000:00000010:0.1:1788201478.482498:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576600. 00000080:00000010:2.1:1788201478.492132:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab57b9c0. 00000080:00000010:2.1:1788201478.492366:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:0.1:1788201478.500208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07000. 00010000:00000010:0.1:1788201478.566173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b500. 00010000:00000010:0.1:1788201478.566180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0c00. 00000080:00000010:0.1:1788201478.570192:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c139c0. 00000080:00000010:0.1:1788201478.570205:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014e0. 00010000:00000010:0.1:1788201478.570213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.593627:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b800. 00010000:00000010:0.1:1788201478.593642:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00000080:00000010:0.1:1788201478.613154:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34ad280. 00000080:00000010:0.1:1788201478.613187:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:0.1:1788201478.621136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.669178:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b200. 00010000:00000010:0.1:1788201478.669192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00000080:00000010:0.1:1788201478.678319:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa885280. 00000080:00000010:0.1:1788201478.678351:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc60. 00010000:00000010:0.1:1788201478.700874:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.930042:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bb00. 00010000:00000010:0.1:1788201478.930057:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1400. 00000080:00000010:0.1:1788201478.943673:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5425ac0. 00000080:00000010:0.1:1788201478.943722:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201820. 00010000:00000010:0.1:1788201478.943733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.954179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8bf00. 00010000:00000010:0.1:1788201478.954193:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00000080:00000010:0.1:1788201478.963793:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc62100. 00000080:00000010:0.1:1788201478.963821:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd60. 00010000:00000010:0.1:1788201478.978138:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201478.986305:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b300. 00010000:00000010:0.1:1788201478.986310:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1000. 00000080:00000010:0.1:1788201478.995682:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3575280. 00000080:00000010:0.1:1788201478.995712:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ccc0. 00010000:00000010:0.1:1788201479.004300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201479.014234:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b500. 00010000:00000010:0.1:1788201479.014241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00000080:00000010:0.1:1788201479.036695:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa905ac0. 00000080:00000010:0.1:1788201479.036792:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c760. 00010000:00000010:0.1:1788201479.040225:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201479.099579:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b000. 00010000:00000010:0.1:1788201479.099585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00000080:00000010:0.1:1788201479.193205:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa49080. 00000080:00000010:0.1:1788201479.193232:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172ce0. 00010000:00000010:0.1:1788201479.207582:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201479.257155:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b600. 00010000:00000010:0.1:1788201479.257160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00010000:00000010:0.1:1788201479.373179:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8be00. 00010000:00000010:0.1:1788201479.373191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697600. 00010000:00000010:0.1:1788201479.427141:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86f00. 00010000:00000010:0.1:1788201479.427155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14200. 00000080:00000010:0.1:1788201479.437206:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf9b180. 00000080:00000010:0.1:1788201479.437249:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a20. 00010000:00000010:0.1:1788201479.449528:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201479.460265:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86700. 00010000:00000010:0.1:1788201479.460276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15400. 00000080:00000010:0.1:1788201479.469149:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84825ac0. 00000080:00000010:0.1:1788201479.469184:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c760. 00010000:00000010:0.1:1788201479.559414:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86200. 00010000:00000010:0.1:1788201479.559422:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15000. 00000080:00000010:0.1:1788201479.566148:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa20000. 00000080:00000010:0.1:1788201479.566163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201be0. 00010000:00000010:0.1:1788201479.578507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:0.1:1788201479.596312:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b900. 00010000:00000010:0.1:1788201479.596326:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15200. 00010000:00000010:0.1:1788201479.648439:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b800. 00010000:00000010:0.1:1788201479.648447:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14e00. 00000080:00000010:0.1:1788201479.665340:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab50eb40. 00000080:00000010:0.1:1788201479.665362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201c20. 00010000:00000010:0.1:1788201479.669217:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:2.1:1788201479.688226:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86000. 00010000:00000010:2.1:1788201479.688239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14a00. 00000080:00000010:2.1:1788201479.705214:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c16300. 00000080:00000010:2.1:1788201479.705240:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8a0. 00010000:00000010:2.1:1788201479.716135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b82d31200. 00010000:00000010:0.1:1788201479.769332:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86d00. 00010000:00000010:0.1:1788201479.769345:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14800. 00000080:00000010:0.1:1788201479.787317:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbdcc200. 00000080:00000010:0.1:1788201479.787344:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981726a0. 00010000:00000010:0.1:1788201479.904250:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045000. 00010000:00000010:0.1:1788201479.920733:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7700. 00010000:00000010:0.1:1788201479.920742:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14c00. 00000080:00000010:0.1:1788201479.931856:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3578840. 00000080:00000010:0.1:1788201479.931880:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201060. 00010000:00000010:0.1:1788201479.935137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201480.053727:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7900. 00010000:00000010:0.1:1788201480.053738:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15600. 00000080:00000010:0.1:1788201480.061147:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa354d280. 00000080:00000010:0.1:1788201480.061177:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52019a0. 00010000:00000010:0.1:1788201480.067909:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201480.211183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7000. 00010000:00000010:0.1:1788201480.211194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eee00. 00000080:00000010:0.1:1788201480.244386:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54118c0. 00000080:00000010:0.1:1788201480.244427:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b80. 00010000:00000010:0.1:1788201480.273384:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b84d8b100. 00010000:00000010:0.1:1788201480.273391:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee200. 00000080:00000010:0.1:1788201480.284332:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa36300. 00000080:00000010:0.1:1788201480.284371:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca60. 00010000:00000010:0.1:1788201480.295672:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:0.1:1788201480.320758:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec300. 00010000:00000010:0.1:1788201480.320767:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efa00. 00000080:00000010:0.1:1788201480.325254:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf39080. 00000080:00000010:0.1:1788201480.325284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c6e0. 00010000:00000010:0.1:1788201480.392193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec600. 00010000:00000010:0.1:1788201480.392209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00000080:00000010:0.1:1788201480.405418:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa545a940. 00000080:00000010:0.1:1788201480.405450:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201fe0. 00010000:00000010:0.1:1788201480.417153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:0.1:1788201480.464277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d900. 00010000:00000010:0.1:1788201480.464290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00010000:00000010:0.1:1788201480.476322:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d500. 00010000:00000010:0.1:1788201480.476328:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00000080:00000010:0.1:1788201480.483951:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3558000. 00000080:00000010:0.1:1788201480.483980:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201760. 00010000:00000010:0.1:1788201480.497262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:0.1:1788201480.507771:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737df00. 00010000:00000010:0.1:1788201480.507779:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef000. 00000080:00000010:0.1:1788201480.517318:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b84b218c0. 00000080:00000010:0.1:1788201480.517360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c260. 00010000:00000010:0.1:1788201480.531228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:2.1:1788201480.659302:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7400. 00010000:00000010:2.1:1788201480.659317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eea00. 00000080:00000010:2.1:1788201480.666212:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54deb40. 00000080:00000010:2.1:1788201480.666261:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52015c0. 00010000:00000010:2.1:1788201480.681573:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d000. 00010000:00000010:2.1:1788201480.694258:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7100. 00010000:00000010:2.1:1788201480.694268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee800. 00000080:00000010:2.1:1788201480.698645:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37598c0. 00000080:00000010:2.1:1788201480.698664:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468da0. 00010000:00000010:2.1:1788201480.703745:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d000. 00010000:00000010:2.1:1788201480.721593:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b90dc7f00. 00010000:00000010:2.1:1788201480.721600:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00000080:00000010:0.1:1788201480.726371:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8590dac0. 00000080:00000010:0.1:1788201480.726392:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468940. 00010000:00000010:0.1:1788201480.833692:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84500. 00010000:00000010:0.1:1788201480.833699:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24c00. 00000080:00000010:0.1:1788201480.840516:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3562100. 00000080:00000010:0.1:1788201480.840547:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201da0. 00010000:00000010:0.1:1788201480.864302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:0.1:1788201480.889146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c400. 00010000:00000010:0.1:1788201480.912289:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84200. 00010000:00000010:0.1:1788201480.912298:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24400. 00000080:00000010:0.1:1788201480.922268:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab61f380. 00000080:00000010:0.1:1788201480.922303:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201480. 00010000:00000010:0.1:1788201480.956682:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:0.1:1788201480.976146:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84300. 00010000:00000010:0.1:1788201480.976155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25e00. 00000080:00000010:0.1:1788201480.982794:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36cb9c0. 00000080:00000010:0.1:1788201480.982816:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c040. 00010000:00000010:0.1:1788201480.992141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d200. 00010000:00000010:0.1:1788201481.050327:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84000. 00010000:00000010:0.1:1788201481.050338:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25800. 00000080:00000010:0.1:1788201481.058867:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e0b9c0. 00000080:00000010:0.1:1788201481.062734:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52016e0. 00010000:00000010:0.1:1788201481.082081:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:2.1:1788201481.118307:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84700. 00010000:00000010:2.1:1788201481.118316:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24600. 00010000:00000010:2.1:1788201481.149508:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6400. 00010000:00000010:2.1:1788201481.200468:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84400. 00010000:00000010:2.1:1788201481.200486:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25a00. 00000080:00000010:0.1:1788201481.228147:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cba940. 00000080:00000010:0.1:1788201481.228183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a981726a0. 00010000:00000010:0.1:1788201481.242340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:2.1:1788201481.303174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84f00. 00010000:00000010:2.1:1788201481.303189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25600. 00010000:00000010:0.1:1788201481.387248:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84d00. 00010000:00000010:0.1:1788201481.387263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24e00. 00000080:00000010:0.1:1788201481.409820:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8594e300. 00000080:00000010:0.1:1788201481.409851:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201700. 00010000:00000010:0.1:1788201481.409860:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:0.1:1788201481.422407:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84e00. 00010000:00000010:0.1:1788201481.422421:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25000. 00000080:00000010:0.1:1788201481.444282:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d86300. 00000080:00000010:0.1:1788201481.444297:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c4c0. 00010000:00000010:2.1:1788201481.453525:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:2.1:1788201481.494329:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84c00. 00010000:00000010:2.1:1788201481.494416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25400. 00000080:00000010:2.1:1788201481.525670:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eeb180. 00000080:00000010:2.1:1788201481.525708:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb00. 00010000:00000010:2.1:1788201481.539159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d800. 00010000:00000010:0.1:1788201481.574828:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84100. 00010000:00000010:0.1:1788201481.574848:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030600. 00000080:00000010:0.1:1788201481.583185:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36fd280. 00000080:00000010:0.1:1788201481.583224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468080. 00010000:00000010:0.1:1788201481.606236:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00010000:00000010:0.1:1788201481.625792:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84900. 00010000:00000010:0.1:1788201481.625811:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00000080:00000010:0.1:1788201481.625823:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab590840. 00000080:00000010:0.1:1788201481.625851:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cde0. 00010000:00000010:0.1:1788201481.635242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00000080:00000010:2.1:1788201481.706595:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa37ae300. 00000080:00000010:2.1:1788201481.706634:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd00. 00010000:00000010:2.1:1788201481.729140:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d800. 00010000:00000010:2.1:1788201481.760193:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d700. 00010000:00000010:2.1:1788201481.760209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030e00. 00000080:00000010:2.1:1788201481.769898:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa04200. 00000080:00000010:2.1:1788201481.769916:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468b40. 00010000:00000010:2.1:1788201481.787524:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d800. 00010000:00000010:2.1:1788201481.857424:0:256136:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:2.1:1788201481.857437:0:256136:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030800. 00010000:00000010:2.1:1788201481.857443:0:256136:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031400. 00000080:00000010:2.1:1788201481.864170:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3702100. 00000080:00000010:2.1:1788201481.864197:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468ea0. 00010000:00000010:0.1:1788201481.911289:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b600. 00010000:00000010:0.1:1788201481.911304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07c00. 00000080:00000010:0.1:1788201481.927171:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3716300. 00000080:00000010:0.1:1788201481.927180:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468b40. 00010000:00000010:0.1:1788201481.936151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201481.962168:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bf00. 00010000:00000010:0.1:1788201481.962183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06400. 00000080:00000010:0.1:1788201481.980233:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbdf8000. 00000080:00000010:0.1:1788201481.980280:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cde0. 00010000:00000010:0.1:1788201482.009242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201482.314158:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96b500. 00010000:00000010:0.1:1788201482.314173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07600. 00000080:00000010:0.1:1788201482.337754:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81640000. 00000080:00000010:0.1:1788201482.337792:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52017a0. 00010000:00000010:0.1:1788201482.350336:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201482.540674:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bd00. 00010000:00000010:0.1:1788201482.540688:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149d800. 00000080:00000010:0.1:1788201482.559163:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa3c200. 00000080:00000010:0.1:1788201482.559192:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e00. 00010000:00000010:0.1:1788201482.570224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201482.598211:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7a8b96bb00. 00010000:00000010:0.1:1788201482.598226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149ca00. 00000080:00000010:2.1:1788201482.611136:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5cfa100. 00000080:00000010:2.1:1788201482.611166:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ccc0. 00010000:00000010:0.1:1788201482.667118:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ece00. 00010000:00000010:0.1:1788201482.667130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149d000. 00000080:00000010:0.1:1788201482.672847:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e62100. 00000080:00000010:0.1:1788201482.672858:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468760. 00010000:00000010:0.1:1788201482.689303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201482.712222:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec100. 00010000:00000010:0.1:1788201482.712229:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149d200. 00000080:00000010:0.1:1788201482.720254:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5492100. 00000080:00000010:0.1:1788201482.720287:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd60. 00010000:00000010:0.1:1788201482.729134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201482.756341:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec200. 00010000:00000010:0.1:1788201482.756353:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149c000. 00000080:00000010:0.1:1788201482.762296:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81486300. 00000080:00000010:0.1:1788201482.762313:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cee0. 00010000:00000010:0.1:1788201482.768262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201482.800354:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec000. 00010000:00000010:0.1:1788201482.800377:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149c800. 00000080:00000010:0.1:1788201482.814755:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56bca40. 00000080:00000010:0.1:1788201482.815652:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd60. 00010000:00000010:0.1:1788201482.821156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201482.847294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec900. 00010000:00000010:0.1:1788201482.847309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871c00. 00000080:00000010:0.1:1788201482.866538:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3712940. 00000080:00000010:0.1:1788201482.866573:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc60. 00010000:00000010:0.1:1788201482.871135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201482.939209:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ec500. 00010000:00000010:0.1:1788201482.939222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00000080:00000010:0.1:1788201482.946811:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe5a100. 00000080:00000010:0.1:1788201482.946835:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018c0. 00010000:00000010:0.1:1788201482.958210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:2.1:1788201482.979161:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecc00. 00010000:00000010:2.1:1788201482.979173:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871e00. 00000080:00000010:0.1:1788201482.982246:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e32940. 00000080:00000010:0.1:1788201482.982266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc60. 00010000:00000010:0.1:1788201483.002538:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.015574:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecb00. 00010000:00000010:0.1:1788201483.015585:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871800. 00000080:00000010:0.1:1788201483.020144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5696b40. 00000080:00000010:0.1:1788201483.020170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:0.1:1788201483.042133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.046313:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b869ecd00. 00010000:00000010:0.1:1788201483.046321:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870a00. 00000080:00000010:0.1:1788201483.057140:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa36a8000. 00000080:00000010:0.1:1788201483.126193:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c498c0. 00000080:00000010:0.1:1788201483.126222:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52018c0. 00010000:00000010:0.1:1788201483.132717:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.149442:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b300. 00010000:00000010:0.1:1788201483.149449:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871400. 00000080:00000010:0.1:1788201483.152537:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3462940. 00000080:00000010:0.1:1788201483.152568:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c480. 00010000:00000010:0.1:1788201483.157155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.166297:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b600. 00010000:00000010:0.1:1788201483.166312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871a00. 00000080:00000010:0.1:1788201483.176918:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab57dac0. 00000080:00000010:0.1:1788201483.176959:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cec0. 00010000:00000010:0.1:1788201483.182904:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.246407:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44e00. 00010000:00000010:0.1:1788201483.246421:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871000. 00000080:00000010:2.1:1788201483.260609:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54fa940. 00000080:00000010:2.1:1788201483.260651:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e00. 00010000:00000010:0.1:1788201483.264181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.398205:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44300. 00010000:00000010:0.1:1788201483.398213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a000. 00010000:00000010:0.1:1788201483.441725:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44000. 00010000:00000010:0.1:1788201483.441737:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00000080:00000010:0.1:1788201483.444917:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5db3180. 00000080:00000010:0.1:1788201483.444952:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201900. 00010000:00000010:0.1:1788201483.457219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00000080:00000010:0.1:1788201483.518546:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3538000. 00000080:00000010:0.1:1788201483.518557:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201700. 00010000:00000010:0.1:1788201483.527205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.543225:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44f00. 00010000:00000010:0.1:1788201483.543238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ac00. 00000080:00000010:0.1:1788201483.549286:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d839c0. 00000080:00000010:0.1:1788201483.549310:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfa0. 00010000:00000010:0.1:1788201483.562192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.588181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016b00. 00010000:00000010:0.1:1788201483.588197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ae00. 00000080:00000010:0.1:1788201483.597037:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3494200. 00000080:00000010:0.1:1788201483.597077:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c9e0. 00010000:00000010:0.1:1788201483.611206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.738334:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016500. 00010000:00000010:0.1:1788201483.738339:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b800. 00000080:00000010:0.1:1788201483.750564:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b85945280. 00000080:00000010:0.1:1788201483.750594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52016e0. 00010000:00000010:0.1:1788201483.750606:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.775679:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016400. 00010000:00000010:0.1:1788201483.775689:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a800. 00000080:00000010:0.1:1788201483.775699:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa540dac0. 00000080:00000010:0.1:1788201483.775725:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc00. 00010000:00000010:0.1:1788201483.779494:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.796673:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016200. 00010000:00000010:0.1:1788201483.796683:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b600. 00000080:00000010:0.1:1788201483.800558:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa0ca40. 00000080:00000010:0.1:1788201483.800583:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00010000:00000010:0.1:1788201483.818516:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.833172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016c00. 00010000:00000010:0.1:1788201483.833183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b000. 00000080:00000010:0.1:1788201483.843140:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab72a940. 00000080:00000010:0.1:1788201483.843169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:2.1:1788201483.846391:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d800. 00010000:00000010:2.1:1788201483.863338:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016f00. 00010000:00000010:2.1:1788201483.863465:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a200. 00000080:00000010:2.1:1788201483.868249:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8147c200. 00000080:00000010:2.1:1788201483.868320:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c060. 00010000:00000010:2.1:1788201483.878291:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d800. 00010000:00000010:2.1:1788201483.904914:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44100. 00010000:00000010:2.1:1788201483.904929:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b200. 00000080:00000010:0.1:1788201483.913717:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56f1080. 00000080:00000010:0.1:1788201483.913745:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468860. 00010000:00000010:0.1:1788201483.921981:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201483.926294:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b82b44800. 00010000:00000010:0.1:1788201483.926309:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19aa00. 00000080:00000010:0.1:1788201483.933387:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e50840. 00000080:00000010:0.1:1788201483.933424:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cb20. 00010000:00000010:0.1:1788201483.940317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:2.1:1788201484.078293:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016900. 00010000:00000010:2.1:1788201484.078306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b82d30c00. 00010000:00000010:0.1:1788201484.119357:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016000. 00010000:00000010:0.1:1788201484.119364:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b82d30e00. 00000080:00000010:0.1:1788201484.141704:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5c23180. 00000080:00000010:0.1:1788201484.141720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172960. 00010000:00000010:0.1:1788201484.147253:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201484.152690:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016100. 00010000:00000010:0.1:1788201484.152697:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b82d30600. 00000080:00000010:0.1:1788201484.162582:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817a8000. 00000080:00000010:0.1:1788201484.162620:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:0.1:1788201484.182064:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:2.1:1788201484.220601:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b8909e600. 00010000:00000010:2.1:1788201484.220617:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b82d31e00. 00000080:00000010:2.1:1788201484.238144:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbca39c0. 00000080:00000010:2.1:1788201484.238164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ca20. 00010000:00000010:0.1:1788201484.249073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9dc00. 00010000:00000010:0.1:1788201484.369133:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201484.397768:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016300. 00010000:00000010:0.1:1788201484.397778:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b82d31600. 00000080:00000010:0.1:1788201484.403547:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a817fa100. 00000080:00000010:0.1:1788201484.403558:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201100. 00010000:00000010:0.1:1788201484.403563:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201484.429180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9e016700. 00010000:00000010:0.1:1788201484.429185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b82d31000. 00000080:00000010:0.1:1788201484.433257:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a815ceb40. 00000080:00000010:0.1:1788201484.433274:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c520. 00010000:00000010:0.1:1788201484.433279:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201484.448752:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7700. 00010000:00000010:0.1:1788201484.448763:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b82d30200. 00000080:00000010:0.1:1788201484.454873:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbc3ca40. 00000080:00000010:0.1:1788201484.454899:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c3a0. 00010000:00000010:0.1:1788201484.472507:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:0.1:1788201484.549587:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4700. 00010000:00000010:0.1:1788201484.549602:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00000080:00000010:0.1:1788201484.577651:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfba940. 00000080:00000010:0.1:1788201484.577678:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201480. 00010000:00000010:0.1:1788201484.588853:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201484.866291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c500. 00010000:00000010:0.1:1788201484.866303:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044c00. 00000080:00000010:0.1:1788201484.877379:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8594a940. 00000080:00000010:0.1:1788201484.877399:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f00. 00010000:00000010:0.1:1788201484.887197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00000080:00000010:0.1:1788201484.914495:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5640000. 00000080:00000010:0.1:1788201484.914505:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:0.1:1788201484.926546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201484.941277:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3cd00. 00010000:00000010:0.1:1788201484.941300:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045c00. 00000080:00000010:0.1:1788201484.961919:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57839c0. 00000080:00000010:0.1:1788201484.961955:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00010000:00000010:0.1:1788201484.969146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201484.996310:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c400. 00010000:00000010:0.1:1788201484.996317:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25400. 00000080:00000010:0.1:1788201485.001898:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3487380. 00000080:00000010:0.1:1788201485.001929:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:2.1:1788201485.100299:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3ca00. 00010000:00000010:2.1:1788201485.100306:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25600. 00000080:00000010:0.1:1788201485.114163:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbef39c0. 00000080:00000010:0.1:1788201485.114201:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b60. 00010000:00000010:0.1:1788201485.130927:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.147175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7800. 00010000:00000010:0.1:1788201485.147185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25a00. 00000080:00000010:0.1:1788201485.147193:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa56d0000. 00000080:00000010:0.1:1788201485.147212:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c7e0. 00010000:00000010:0.1:1788201485.162186:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.186194:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7600. 00010000:00000010:0.1:1788201485.186203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24600. 00010000:00000010:0.1:1788201485.224346:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7900. 00010000:00000010:0.1:1788201485.224355:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25800. 00000080:00000010:0.1:1788201485.224381:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa544b180. 00000080:00000010:0.1:1788201485.224403:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201e40. 00010000:00000010:0.1:1788201485.232233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.288797:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22c00. 00010000:00000010:0.1:1788201485.288805:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25e00. 00000080:00000010:0.1:1788201485.296662:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab71e300. 00000080:00000010:0.1:1788201485.296689:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b80. 00010000:00000010:0.1:1788201485.300592:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.348317:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22400. 00010000:00000010:0.1:1788201485.348325:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25c00. 00000080:00000010:0.1:1788201485.358197:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e32100. 00000080:00000010:0.1:1788201485.358242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201840. 00010000:00000010:0.1:1788201485.368530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.392254:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22900. 00010000:00000010:0.1:1788201485.392261:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24c00. 00000080:00000010:0.1:1788201485.409156:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5664a40. 00000080:00000010:0.1:1788201485.409187:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c8c0. 00010000:00000010:0.1:1788201485.409194:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.415337:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22600. 00010000:00000010:0.1:1788201485.415345:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24200. 00000080:00000010:0.1:1788201485.434460:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f34a40. 00000080:00000010:0.1:1788201485.434492:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf40. 00010000:00000010:2.1:1788201485.437461:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fc00. 00010000:00000010:0.1:1788201485.454863:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22100. 00010000:00000010:0.1:1788201485.454877:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24000. 00000080:00000010:0.1:1788201485.454887:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57eeb40. 00000080:00000010:0.1:1788201485.454909:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:0.1:1788201485.467979:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.530488:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fc00. 00010000:00000010:0.1:1788201485.562757:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7500. 00010000:00000010:0.1:1788201485.562776:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee800. 00000080:00000010:0.1:1788201485.583055:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa54c8840. 00000080:00000010:0.1:1788201485.583084:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201780. 00010000:00000010:0.1:1788201485.583112:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.590375:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7300. 00010000:00000010:0.1:1788201485.590388:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eea00. 00000080:00000010:0.1:1788201485.599134:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf6ca40. 00000080:00000010:0.1:1788201485.599164:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:0.1:1788201485.609245:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.623461:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7000. 00010000:00000010:0.1:1788201485.623469:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eec00. 00000080:00000010:0.1:1788201485.628177:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa346dac0. 00000080:00000010:0.1:1788201485.628198:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2c0. 00010000:00000010:0.1:1788201485.640057:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.658107:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7400. 00010000:00000010:0.1:1788201485.658119:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee600. 00000080:00000010:0.1:1788201485.667830:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfc6b40. 00000080:00000010:0.1:1788201485.667871:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c640. 00010000:00000010:0.1:1788201485.673125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.682081:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7f00. 00010000:00000010:0.1:1788201485.682087:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef200. 00000080:00000010:0.1:1788201485.693027:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa22940. 00000080:00000010:0.1:1788201485.693055:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfc0. 00010000:00000010:0.1:1788201485.703075:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.715174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7200. 00010000:00000010:0.1:1788201485.715187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef000. 00000080:00000010:0.1:1788201485.729145:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3725280. 00000080:00000010:0.1:1788201485.729169:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2c0. 00010000:00000010:0.1:1788201485.737363:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.769175:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7e00. 00010000:00000010:0.1:1788201485.769187:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:0.1:1788201485.834434:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22a00. 00010000:00000010:0.1:1788201485.834453:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00000080:00000010:0.1:1788201485.861148:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaaa45ac0. 00000080:00000010:0.1:1788201485.861183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201060. 00010000:00000010:0.1:1788201485.869848:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201485.911214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22f00. 00010000:00000010:0.1:1788201485.911228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00000080:00000010:0.1:1788201485.925192:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa353a940. 00000080:00000010:0.1:1788201485.925224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c2c0. 00010000:00000010:0.1:1788201485.936141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:2.1:1788201485.971194:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22b00. 00010000:00000010:2.1:1788201485.971200:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef600. 00000080:00000010:2.1:1788201485.978151:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5644a40. 00000080:00000010:2.1:1788201485.978167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfc0. 00010000:00000010:2.1:1788201485.987087:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f800. 00010000:00000010:2.1:1788201486.008922:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22200. 00010000:00000010:2.1:1788201486.008930:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efa00. 00000080:00000010:2.1:1788201486.008949:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a814698c0. 00000080:00000010:2.1:1788201486.008968:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468c20. 00010000:00000010:2.1:1788201486.017922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f800. 00000080:00000010:0.1:1788201486.038652:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab642940. 00000080:00000010:0.1:1788201486.038673:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468560. 00010000:00000010:0.1:1788201486.044149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201486.069060:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22000. 00010000:00000010:0.1:1788201486.069069:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eee00. 00000080:00000010:0.1:1788201486.077200:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a8164b9c0. 00000080:00000010:0.1:1788201486.077226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c400. 00010000:00000010:0.1:1788201486.083424:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201486.101244:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b94a22d00. 00010000:00000010:0.1:1788201486.101251:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00000080:00000010:0.1:1788201486.109269:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbdb2100. 00000080:00000010:0.1:1788201486.109322:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd40. 00010000:00000010:0.1:1788201486.114444:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201486.125236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7c00. 00010000:00000010:0.1:1788201486.125245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06c00. 00000080:00000010:0.1:1788201486.130517:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa3614a40. 00000080:00000010:0.1:1788201486.130547:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce20. 00010000:00000010:0.1:1788201486.144510:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201486.180292:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850800. 00010000:00000010:0.1:1788201486.180299:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576600. 00000080:00000010:0.1:1788201486.187259:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa34f9080. 00000080:00000010:0.1:1788201486.187295:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172ee0. 00010000:00000010:0.1:1788201486.192257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031e00. 00010000:00000010:0.1:1788201486.641473:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850100. 00010000:00000010:0.1:1788201486.641479:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5200. 00000080:00000010:0.1:1788201486.648267:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa35fb9c0. 00000080:00000010:0.1:1788201486.648286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201a20. 00010000:00000010:0.1:1788201486.664149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201486.691179:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850200. 00010000:00000010:0.1:1788201486.691188:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5600. 00000080:00000010:0.1:1788201486.705087:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81500840. 00000080:00000010:0.1:1788201486.705451:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cc40. 00010000:00000010:0.1:1788201486.717152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201486.815318:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c300. 00010000:00000010:0.1:1788201486.815327:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5800. 00000080:00000010:0.1:1788201486.824211:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5477380. 00000080:00000010:0.1:1788201486.824240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014a0. 00010000:00000010:0.1:1788201486.832138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201486.934218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c500. 00010000:00000010:0.1:1788201486.934229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7c00. 00000080:00000010:0.1:1788201486.939942:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8590c200. 00000080:00000010:0.1:1788201486.939972:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201be0. 00010000:00000010:0.1:1788201486.943433:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201486.966812:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ba11f7b00. 00010000:00000010:0.1:1788201486.966822:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7e00. 00000080:00000010:0.1:1788201486.971225:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81534200. 00000080:00000010:0.1:1788201486.971255:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:0.1:1788201486.981269:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.001048:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850c00. 00010000:00000010:0.1:1788201487.001063:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c200. 00010000:00000010:0.1:1788201487.109165:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c800. 00010000:00000010:0.1:1788201487.109177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c000. 00000080:00000010:0.1:1788201487.115674:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57ca100. 00000080:00000010:0.1:1788201487.115705:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201820. 00010000:00000010:0.1:1788201487.124239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.142183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fe00. 00010000:00000010:0.1:1788201487.142196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c400. 00000080:00000010:0.1:1788201487.151265:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa57339c0. 00000080:00000010:0.1:1788201487.151319:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:0.1:1788201487.161240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.184550:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f900. 00010000:00000010:0.1:1788201487.184557:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d400. 00000080:00000010:0.1:1788201487.191216:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf939c0. 00000080:00000010:0.1:1788201487.191240:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf20. 00010000:00000010:0.1:1788201487.195245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.214455:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f000. 00010000:00000010:0.1:1788201487.214467:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9da00. 00000080:00000010:0.1:1788201487.218256:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f8c200. 00000080:00000010:0.1:1788201487.218265:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:0.1:1788201487.233402:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.240182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850e00. 00010000:00000010:0.1:1788201487.240197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9cc00. 00000080:00000010:0.1:1788201487.255931:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab6fe300. 00000080:00000010:0.1:1788201487.255954:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cbc0. 00010000:00000010:0.1:1788201487.260191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.277956:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850900. 00010000:00000010:0.1:1788201487.277962:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9ce00. 00000080:00000010:2.1:1788201487.284681:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7b8403dac0. 00000080:00000010:2.1:1788201487.284709:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cf20. 00010000:00000010:2.1:1788201487.292141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e400. 00010000:00000010:2.1:1788201487.310265:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850000. 00010000:00000010:2.1:1788201487.310288:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c600. 00000080:00000010:2.1:1788201487.328246:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbf00840. 00000080:00000010:2.1:1788201487.328539:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468fe0. 00010000:00000010:2.1:1788201487.328547:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e400. 00010000:00000010:0.1:1788201487.339160:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850300. 00010000:00000010:0.1:1788201487.339172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d800. 00000080:00000010:0.1:1788201487.345900:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe0a940. 00000080:00000010:0.1:1788201487.345917:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468e00. 00010000:00000010:0.1:1788201487.352197:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.366270:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f300. 00010000:00000010:0.1:1788201487.366302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9de00. 00000080:00000010:0.1:1788201487.372019:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aaa9ff380. 00000080:00000010:0.1:1788201487.372054:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:0.1:1788201487.374223:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.395721:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f700. 00010000:00000010:0.1:1788201487.395733:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9dc00. 00000080:00000010:0.1:1788201487.404779:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d80000. 00000080:00000010:0.1:1788201487.404801:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cbc0. 00010000:00000010:0.1:1788201487.412189:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.419228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f400. 00010000:00000010:0.1:1788201487.419242:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c800. 00000080:00000010:0.1:1788201487.425439:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7a81599080. 00000080:00000010:0.1:1788201487.425460:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c440. 00010000:00000010:0.1:1788201487.436872:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031800. 00010000:00000010:0.1:1788201487.446273:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f500. 00010000:00000010:0.1:1788201487.446302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d600. 00000080:00000010:0.1:1788201487.465285:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbec1080. 00000080:00000010:0.1:1788201487.465316:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:2.1:1788201487.469558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00010000:00000010:0.1:1788201487.497644:0:12:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f800. 00010000:00000010:0.1:1788201487.497651:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d000. 00000080:00000010:0.1:1788201487.506510:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab559080. 00000080:00000010:0.1:1788201487.506529:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894681a0. 00010000:00000010:0.1:1788201487.517601:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201487.523762:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fa00. 00010000:00000010:0.1:1788201487.523774:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbda5dc00. 00000080:00000010:2.1:1788201487.531138:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbe09080. 00000080:00000010:2.1:1788201487.531161:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c580. 00010000:00000010:2.1:1788201487.555154:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fd00. 00010000:00000010:2.1:1788201487.555163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbda5d800. 00000080:00000010:2.1:1788201487.559139:0:408066:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5788000. 00000080:00000010:2.1:1788201487.559264:0:408066:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468fe0. 00010000:00000010:2.1:1788201487.569160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f600. 00010000:00000010:2.1:1788201487.579126:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fb00. 00010000:00000010:2.1:1788201487.579133:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbda5c800. 00000080:00000010:2.1:1788201487.589203:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab76a100. 00000080:00000010:2.1:1788201487.589229:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468120. 00010000:00000010:2.1:1788201487.597127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f600. 00010000:00000010:0.1:1788201487.622134:0:408066:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f100. 00010000:00000010:0.1:1788201487.622140:0:408066:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbda5d600. 00010000:00000010:0.1:1788201487.662339:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f200. 00010000:00000010:0.1:1788201487.662350:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbda5d000. 00000080:00000010:2.1:1788201487.670328:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5df9080. 00000080:00000010:2.1:1788201487.670360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab52014e0. 00010000:00000010:0.1:1788201487.678153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f800. 00010000:00000010:0.1:1788201487.740235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00010000:00000010:0.1:1788201487.775182:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850700. 00010000:00000010:0.1:1788201487.775197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbda5d200. 00000080:00000010:0.1:1788201487.785267:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab51a940. 00000080:00000010:0.1:1788201487.785342:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201740. 00000080:00000010:2.1:1788201487.863225:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa561b180. 00000080:00000010:2.1:1788201487.863273:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c180. 00010000:00000010:0.1:1788201487.885378:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:0.1:1788201487.915505:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3850500. 00010000:00000010:0.1:1788201487.915517:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbda5c000. 00000080:00000010:0.1:1788201487.925153:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5647380. 00000080:00000010:0.1:1788201487.925196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a894680a0. 00010000:00000010:0.1:1788201487.935267:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:0.1:1788201488.022525:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858d00. 00010000:00000010:0.1:1788201488.022537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbda5c400. 00000080:00000010:0.1:1788201488.050151:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7abbfbca40. 00000080:00000010:0.1:1788201488.050189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b20. 00010000:00000010:0.1:1788201488.216442:0:408155:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5c00. 00010000:00000010:0.1:1788201488.216457:0:408155:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d100. 00010000:00000010:0.1:1788201488.216461:0:408155:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24800. 00010000:00000010:0.1:1788201488.216468:0:408155:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7ab3952a00. 00010000:00000010:0.1:1788201488.216470:0:408155:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5000. 00000080:00000010:0.1:1788201488.231709:0:408155:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5f25ac0. 00000080:00000010:0.1:1788201488.231730:0:408155:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172780. 00010000:00000010:0.1:1788201488.264178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00000080:00000010:0.1:1788201488.267336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd40. 00010000:00000010:0.1:1788201488.267350:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d100. 00010000:00000010:0.1:1788201488.267355:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031c00. 00010000:00000010:0.1:1788201489.983964:0:408165:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030600. 00000080:00000010:0.1:1788201489.983977:0:408165:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce80. 00010000:00000010:0.1:1788201490.004133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00000080:00000010:0.1:1788201490.004140:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cae0. 00010000:00000010:2.1:1788201490.093536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:2.1:1788201490.093544:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7800. 00000080:00000010:2.1:1788201490.093548:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a98172ec0. 00010000:00000010:2.1:1788201490.105122:0:408165:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e400. 00010000:00000010:2.1:1788201490.128205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e400. 00000080:00000010:2.1:1788201490.128219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468a40. 00010000:00000010:2.1:1788201490.149590:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:2.1:1788201490.149605:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468de0. 00010000:00000010:2.1:1788201490.149616:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00010000:00000010:2.1:1788201490.170642:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e400. 00000080:00000010:2.1:1788201490.170658:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a89468a40. 00000080:00000010:1.1:1788201490.214203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201680. 00010000:00000010:1.1:1788201490.219584:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5e00. 00010000:00000010:1.1:1788201490.225439:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4400. 00000080:00000010:1.1:1788201490.225457:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201680. 00010000:00000010:1.1:1788201490.235138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5e00. 00000080:00000010:1.1:1788201490.235144:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b00. 00010000:00000010:0.1:1788201490.240139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4400. 00010000:00000010:0.1:1788201490.260934:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030a00. 00000080:00000010:0.1:1788201490.260953:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cae0. 00010000:00000010:0.1:1788201490.264410:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00000080:00000010:0.1:1788201490.264420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3ce80. 00010000:00000010:0.1:1788201490.306375:0:1353:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031400. 00010000:00000010:1.1:1788201490.346066:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5e00. 00000080:00000010:1.1:1788201490.346078:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201b00. 00010000:00000010:1.1:1788201490.346104:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4c00. 00000080:00000010:1.1:1788201490.346107:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201680. 00010000:00000010:1.1:1788201490.346111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4400. 00010000:00000010:1.1:1788201490.346116:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4a00. 00000080:00000010:1.1:1788201490.346119:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201540. 00010000:00000010:1.1:1788201490.346133:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5400. 00000080:00000010:1.1:1788201490.346136:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201f80. 00010000:00000010:1.1:1788201490.346142:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5a00. 00010000:00000010:1.1:1788201490.367394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4e00. 00000080:00000010:1.1:1788201490.367406:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201d40. 00010000:00000010:1.1:1788201490.367412:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4600. 00000080:00000010:1.1:1788201490.367414:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201580. 00010000:00000010:1.1:1788201490.367417:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4000. 00010000:00000010:0.1:1788201550.209806:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201550.209814:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d4e0. 00010000:00000010:0.1:1788201550.210130:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:2.1:1788201550.215882:0:410432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:2.1:1788201550.215889:0:410432:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b5c0. 00010000:00000010:2.1:1788201550.215895:0:410432:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:2.1:1788201550.215897:0:410432:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54ba40. 00010000:00000010:2.1:1788201550.232706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201550.232718:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0e0. 00010000:00000010:2.1:1788201550.232728:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201550.232731:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db920. 00010000:00000010:0.1:1788201550.233341:0:410439:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35e00. 00000080:00000010:0.1:1788201550.233350:0:410439:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d0e0. 00010000:00000010:0.1:1788201550.233357:0:410439:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00000080:00000010:0.1:1788201550.233359:0:410439:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dac0. 00010000:00000010:1.1:1788201550.240254:0:408173:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:1.1:1788201550.240264:0:408173:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc80. 00010000:00000010:0.1:1788201550.266073:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:0.1:1788201550.266081:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d880. 00010000:00000010:1.1:1788201550.266383:0:410440:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00010000:00000010:2.1:1788201550.268845:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00010000:00000010:0.1:1788201550.275698:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201550.275710:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d4e0. 00010000:00000010:0.1:1788201550.275720:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201550.275725:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db20. 00010000:00000010:1.1:1788201550.288121:0:410443:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00010000:00000010:2.1:1788201550.289934:0:410441:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35e00. 00010000:00000010:2.1:1788201550.289941:0:410441:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00010000:00000010:0.1:1788201550.299711:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:0.1:1788201550.299719:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bfa0. 00010000:00000010:1.1:1788201550.327115:0:410447:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201550.327131:0:410447:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b1e0. 00010000:00000010:1.1:1788201550.351117:0:409198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:1.1:1788201550.351126:0:409198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dac0. 00010000:00000010:2.1:1788201550.352801:0:409126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201550.352811:0:409126:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db4c0. 00010000:00000010:2.1:1788201550.352819:0:409126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:2.1:1788201550.352825:0:409126:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0e0. 00010000:00000010:0.1:1788201550.356701:0:410450:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201550.356711:0:410450:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db20. 00010000:00000010:0.1:1788201550.356720:0:410450:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201550.356722:0:410450:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d880. 00010000:00000010:1.1:1788201550.359115:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00000080:00000010:1.1:1788201550.359121:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b520. 00010000:00000010:0.1:1788201550.362845:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00000080:00000010:0.1:1788201550.362853:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc80. 00010000:00000010:2.1:1788201550.367117:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:2.1:1788201550.367143:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:2.1:1788201550.367147:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf60. 00010000:00000010:1.1:1788201550.380755:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e800. 00010000:00000010:2.1:1788201550.383397:0:409126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:2.1:1788201550.383409:0:409126:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb00. 00010000:00000010:0.1:1788201550.388293:0:410455:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35e00. 00010000:00000010:2.1:1788201550.401058:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00010000:00000010:2.1:1788201550.403795:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00010000:00000010:1.1:1788201550.412838:0:410448:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00010000:00000010:2.1:1788201550.413657:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:2.1:1788201550.413667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf60. 00010000:00000010:1.1:1788201550.447134:0:410458:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e800. 00000080:00000010:1.1:1788201550.447141:0:410458:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b5c0. 00010000:00000010:0.1:1788201550.455270:0:410459:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35e00. 00000080:00000010:0.1:1788201550.455276:0:410459:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d280. 00010000:00000010:2.1:1788201550.469124:0:409126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:2.1:1788201550.469134:0:409126:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0e0. 00010000:00000010:1.1:1788201550.477282:0:410461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00010000:00000010:2.1:1788201550.488078:0:410462:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:2.1:1788201550.488103:0:410462:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbee0. 00000080:00000010:1.1:1788201550.489625:0:410461:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bee0. 00010000:00000010:1.1:1788201550.504499:0:410461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00000080:00000010:1.1:1788201550.510364:0:410461:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b520. 00010000:00000010:2.1:1788201550.510678:0:410465:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00010000:00000010:2.1:1788201550.510688:0:410465:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201550.510692:0:410465:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf60. 00010000:00000010:0.1:1788201550.513751:0:410459:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00000080:00000010:0.1:1788201550.513760:0:410459:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd40. 00010000:00000010:0.1:1788201550.520114:0:409126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:0.1:1788201550.520121:0:409126:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0e0. 00010000:00000010:1.1:1788201550.520598:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e800. 00010000:00000010:2.1:1788201550.529734:0:410466:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:2.1:1788201550.529744:0:410466:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbee0. 00010000:00000010:1.1:1788201550.541616:0:410456:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201550.541624:0:410456:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b540. 00010000:00000010:0.1:1788201550.543062:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:1.1:1788201550.551082:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:1.1:1788201550.551113:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35e00. 00010000:00000010:2.1:1788201550.576121:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00010000:00000010:1.1:1788201550.585493:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00000080:00000010:1.1:1788201550.585500:0:409396:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd40. 00010000:00000010:1.1:1788201550.605149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e800. 00010000:00000010:1.1:1788201550.605158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201550.605162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b520. 00010000:00000010:0.1:1788201550.613058:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201550.613066:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd20. 00010000:00000010:2.1:1788201550.625133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201550.625145:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf60. 00010000:00000010:0.1:1788201550.625192:0:410443:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201550.625199:0:410443:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d220. 00010000:00000010:0.1:1788201550.635118:0:410442:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201550.635133:0:410442:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d6e0. 00010000:00000010:2.1:1788201550.680461:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:2.1:1788201550.680470:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db6c0. 00010000:00000010:1.1:1788201550.692530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201550.692542:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b520. 00010000:00000010:2.1:1788201550.696402:0:410484:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:1.1:1788201550.713238:0:410487:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00010000:00000010:1.1:1788201550.713244:0:410487:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00000080:00000010:1.1:1788201550.713247:0:410487:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b8c0. 00010000:00000010:0.1:1788201550.713352:0:410442:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:0.1:1788201550.713358:0:410442:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00000080:00000010:0.1:1788201550.713362:0:410442:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de80. 00010000:00000010:1.1:1788201550.743402:0:410486:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201550.743411:0:410486:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bfe0. 00010000:00000010:2.1:1788201550.744125:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e800. 00000080:00000010:2.1:1788201550.744134:0:409396:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b4e0. 00010000:00000010:2.1:1788201550.744140:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00010000:00000010:0.1:1788201550.763416:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:2.1:1788201550.774279:0:408173:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201550.774287:0:408173:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db9e0. 00010000:00000010:2.1:1788201550.774294:0:408173:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:0.1:1788201550.775355:0:409126:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de80. 00010000:00000010:1.1:1788201550.783647:0:410491:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00010000:00000010:1.1:1788201550.783653:0:410491:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201550.783656:0:410491:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b8c0. 00010000:00000010:0.1:1788201550.791669:0:408285:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201550.791677:0:408285:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd20. 00010000:00000010:2.1:1788201550.804114:0:409248:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00000080:00000010:2.1:1788201550.804133:0:409248:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db7a0. 00010000:00000010:1.1:1788201550.810471:0:410493:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201550.810479:0:410493:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bbe0. 00010000:00000010:0.1:1788201550.811394:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:2.1:1788201550.819114:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:1.1:1788201550.846129:0:410496:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00010000:00000010:0.1:1788201550.846280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647400. 00010000:00000010:0.1:1788201550.846287:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201550.846293:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd20. 00010000:00000010:2.1:1788201550.847286:0:410495:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201550.863765:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db180. 00010000:00000010:2.1:1788201550.873289:0:410499:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201550.873297:0:410499:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db620. 00010000:00000010:2.1:1788201550.884129:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:2.1:1788201550.884136:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb80. 00010000:00000010:0.1:1788201550.884288:0:410498:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00000080:00000010:0.1:1788201550.884294:0:410498:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d220. 00010000:00000010:0.1:1788201550.884305:0:410498:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201550.899864:0:410501:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de80. 00010000:00000010:2.1:1788201550.914127:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:2.1:1788201550.914136:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db180. 00010000:00000010:2.1:1788201550.914147:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:2.1:1788201550.914150:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd00. 00010000:00000010:2.1:1788201550.914154:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646000. 00000080:00000010:2.1:1788201550.914158:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab539cda0. 00010000:00000010:1.1:1788201550.915384:0:410504:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:1.1:1788201550.915392:0:410504:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd20. 00010000:00000010:0.1:1788201550.932651:0:410507:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00000080:00000010:0.1:1788201550.932664:0:410507:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d480. 00010000:00000010:0.1:1788201550.932673:0:410507:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00010000:00000010:2.1:1788201550.944706:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:2.1:1788201550.944720:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb80. 00010000:00000010:1.1:1788201550.953188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00010000:00000010:0.1:1788201550.953337:0:410505:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201550.955371:0:410505:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d9c0. 00010000:00000010:2.1:1788201550.956060:0:410500:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00010000:00000010:2.1:1788201550.956068:0:410500:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00010000:00000010:2.1:1788201550.956071:0:410500:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:1.1:1788201551.006125:0:409126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00000080:00000010:1.1:1788201551.006130:0:409126:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d480. 00010000:00000010:1.1:1788201551.006138:0:409126:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00010000:00000010:1.1:1788201551.019367:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201551.019374:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b0a0. 00010000:00000010:2.1:1788201551.021067:0:409198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201551.021077:0:409198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb80. 00010000:00000010:2.1:1788201551.021085:0:409198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201551.026288:0:409198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf80. 00010000:00000010:1.1:1788201551.031138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201551.031144:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bf60. 00010000:00000010:2.1:1788201551.041819:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201551.041836:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db560. 00010000:00000010:2.1:1788201551.041843:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201551.041846:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db800. 00010000:00000010:0.1:1788201551.047108:0:410518:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00010000:00000010:2.1:1788201551.047703:0:410519:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646400. 00010000:00000010:1.1:1788201551.061709:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201551.070711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b960. 00010000:00000010:1.1:1788201551.077933:0:410514:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201551.077943:0:410514:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b0a0. 00010000:00000010:1.1:1788201551.077950:0:410514:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00010000:00000010:2.1:1788201551.079325:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00010000:00000010:1.1:1788201551.117129:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201551.117136:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bf60. 00010000:00000010:2.1:1788201551.118111:0:409198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201551.118120:0:409198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbde0. 00010000:00000010:0.1:1788201551.118683:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201551.118691:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3df60. 00010000:00000010:0.1:1788201551.118699:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00000080:00000010:0.1:1788201551.118702:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d9c0. 00010000:00000010:0.1:1788201551.132752:0:410523:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201551.132766:0:410523:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d620. 00010000:00000010:1.1:1788201551.134075:0:410522:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201551.134085:0:410522:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b0a0. 00010000:00000010:2.1:1788201551.150691:0:1032:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201551.150701:0:1032:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbc00. 00010000:00000010:2.1:1788201551.150803:0:1032:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:2.1:1788201551.150807:0:1032:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf80. 00010000:00000010:0.1:1788201551.152708:0:410525:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00000080:00000010:0.1:1788201551.152718:0:410525:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d860. 00010000:00000010:0.1:1788201551.152725:0:410525:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34200. 00010000:00000010:1.1:1788201551.158108:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00010000:00000010:2.1:1788201551.158747:0:410524:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201551.158757:0:410524:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db120. 00010000:00000010:2.1:1788201551.163105:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:2.1:1788201551.187014:0:410528:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00010000:00000010:1.1:1788201551.194816:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35c00. 00010000:00000010:2.1:1788201551.201710:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647e00. 00000080:00000010:2.1:1788201551.206124:0:410504:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab539c6e0. 00010000:00000010:2.1:1788201551.217160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:1.1:1788201551.217652:0:410534:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201551.217660:0:410534:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b400. 00010000:00000010:1.1:1788201551.217668:0:410534:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34200. 00000080:00000010:1.1:1788201551.217673:0:410534:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d860. 00010000:00000010:1.1:1788201551.227690:0:410534:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201551.227696:0:410534:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b740. 00010000:00000010:0.1:1788201551.250571:0:410535:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:0.1:1788201551.250583:0:410535:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b0a0. 00010000:00000010:2.1:1788201551.258613:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201551.258618:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db6c0. 00010000:00000010:1.1:1788201551.264683:0:410536:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201551.264691:0:410536:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bf60. 00010000:00000010:1.1:1788201551.266126:0:410539:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00010000:00000010:1.1:1788201551.298107:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201551.298114:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bb20. 00010000:00000010:2.1:1788201551.300078:0:409198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:2.1:1788201551.300087:0:409198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db540. 00010000:00000010:0.1:1788201551.314073:0:410541:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00010000:00000010:2.1:1788201551.315049:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201551.315059:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf80. 00010000:00000010:1.1:1788201551.321635:0:410543:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00010000:00000010:1.1:1788201551.321643:0:410543:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201551.321648:0:410543:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bf60. 00010000:00000010:2.1:1788201551.324124:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00010000:00000010:2.1:1788201551.334283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201551.334294:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0a0. 00010000:00000010:0.1:1788201551.354466:0:408285:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201551.354477:0:408285:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d800. 00010000:00000010:1.1:1788201551.362432:0:410546:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201551.362439:0:410546:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bb20. 00010000:00000010:0.1:1788201551.368634:0:409348:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00000080:00000010:0.1:1788201551.368642:0:409348:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3da60. 00010000:00000010:2.1:1788201551.371279:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201551.371286:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf80. 00010000:00000010:2.1:1788201551.379480:0:410504:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00010000:00000010:2.1:1788201551.379547:0:410504:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:2.1:1788201551.379552:0:410504:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db3c0. 00010000:00000010:0.1:1788201551.428341:0:409348:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:1.1:1788201551.459932:0:410544:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00010000:00000010:1.1:1788201551.459997:0:410544:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201551.460001:0:410544:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b4a0. 00010000:00000010:0.1:1788201551.471822:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00000080:00000010:0.1:1788201551.471827:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d900. 00010000:00000010:2.1:1788201551.479573:0:410553:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201551.479578:0:410553:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db5e0. 00010000:00000010:1.1:1788201551.480114:0:410496:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201551.480121:0:410496:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b760. 00010000:00000010:2.1:1788201551.491124:0:410553:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201551.491134:0:410553:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf20. 00010000:00000010:0.1:1788201551.493474:0:409348:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00000080:00000010:0.1:1788201551.493481:0:409348:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d100. 00010000:00000010:0.1:1788201551.493487:0:409348:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201551.493490:0:409348:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d6c0. 00010000:00000010:0.1:1788201551.501129:0:410494:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:0.1:1788201551.501141:0:410494:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbe40. 00010000:00000010:1.1:1788201551.512348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00010000:00000010:0.1:1788201551.515464:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00010000:00000010:0.1:1788201551.515473:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00000080:00000010:0.1:1788201551.515478:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d660. 00010000:00000010:2.1:1788201551.537112:0:410557:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:1.1:1788201551.540932:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:1.1:1788201551.540940:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd00. 00010000:00000010:1.1:1788201551.550483:0:409248:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00010000:00000010:0.1:1788201551.551645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647e00. 00010000:00000010:0.1:1788201551.551744:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00010000:00000010:2.1:1788201551.580767:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201551.580776:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db8e0. 00010000:00000010:0.1:1788201551.581712:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647800. 00010000:00000010:1.1:1788201551.598183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00000080:00000010:1.1:1788201551.598191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d9e0. 00000080:00000010:0.1:1788201551.608437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bee0. 00010000:00000010:1.1:1788201551.613118:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00010000:00000010:0.1:1788201551.614205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:0.1:1788201551.614215:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db760. 00010000:00000010:1.1:1788201551.616133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:2.1:1788201551.616137:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bd20. 00010000:00000010:0.1:1788201551.620262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201551.620269:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3daa0. 00010000:00000010:1.1:1788201551.620403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201551.620410:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b0e0. 00010000:00000010:2.1:1788201551.647755:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:2.1:1788201551.647763:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbbe0. 00010000:00000010:0.1:1788201551.648546:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00000080:00000010:0.1:1788201551.648552:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc20. 00010000:00000010:1.1:1788201551.662135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201551.662144:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bc00. 00010000:00000010:1.1:1788201551.666464:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:0.1:1788201551.666537:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34e00. 00010000:00000010:2.1:1788201551.667806:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:1.1:1788201551.682068:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00010000:00000010:0.1:1788201551.682764:0:409248:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201551.682769:0:409248:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3df40. 00010000:00000010:0.1:1788201551.682780:0:409248:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:0.1:1788201551.689262:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0c0. 00010000:00000010:0.1:1788201551.693369:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00010000:00000010:2.1:1788201551.693634:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:2.1:1788201551.693641:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbbe0. 00010000:00000010:1.1:1788201551.699339:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00010000:00000010:1.1:1788201551.699346:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201551.699350:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bc00. 00000080:00000010:0.1:1788201551.699411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc20. 00010000:00000010:2.1:1788201551.716305:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201551.716316:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db000. 00010000:00000010:1.1:1788201551.722237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201551.722244:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b0e0. 00010000:00000010:1.1:1788201551.725270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201551.725276:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b220. 00010000:00000010:0.1:1788201551.728688:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201551.728708:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3daa0. 00010000:00000010:0.1:1788201551.731120:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201551.731126:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3df40. 00010000:00000010:2.1:1788201551.734414:0:410553:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646800. 00010000:00000010:1.1:1788201551.736701:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201551.736714:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bc00. 00010000:00000010:0.1:1788201551.745276:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00010000:00000010:0.1:1788201551.758203:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00000080:00000010:0.1:1788201551.758210:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc20. 00010000:00000010:2.1:1788201551.758313:0:409348:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201551.758321:0:409348:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db160. 00010000:00000010:1.1:1788201551.759131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e800. 00000080:00000010:1.1:1788201551.759136:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b760. 00010000:00000010:1.1:1788201551.766205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35c00. 00010000:00000010:1.1:1788201551.766214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00010000:00000010:2.1:1788201551.782184:0:409348:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201551.782194:0:409348:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db160. 00010000:00000010:2.1:1788201551.790221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00010000:00000010:0.1:1788201551.790449:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00000080:00000010:0.1:1788201551.790456:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc20. 00010000:00000010:1.1:1788201551.807901:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:2.1:1788201551.808133:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db480. 00010000:00000010:1.1:1788201551.814818:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00010000:00000010:1.1:1788201551.814826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00010000:00000010:1.1:1788201551.814830:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:0.1:1788201551.816370:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bc00. 00010000:00000010:0.1:1788201551.816382:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e800. 00000080:00000010:0.1:1788201551.816385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b760. 00010000:00000010:0.1:1788201551.816389:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201551.816394:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3df40. 00010000:00000010:0.1:1788201551.818367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00010000:00000010:2.1:1788201551.818367:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:1.1:1788201551.819080:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b220. 00010000:00000010:1.1:1788201551.821368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00000080:00000010:1.1:1788201551.821385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc20. 00010000:00000010:1.1:1788201551.859183:0:409248:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:1.1:1788201551.859196:0:409248:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbe60. 00010000:00000010:1.1:1788201551.862113:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00010000:00000010:1.1:1788201551.862117:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201551.862121:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bb20. 00010000:00000010:1.1:1788201551.862126:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201551.862129:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b220. 00010000:00000010:0.1:1788201551.862765:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:1.1:1788201551.883110:0:409248:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00010000:00000010:2.1:1788201551.891857:0:410557:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00000080:00000010:2.1:1788201551.891866:0:410557:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3ddc0. 00010000:00000010:1.1:1788201551.902465:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00010000:00000010:0.1:1788201551.921957:0:409198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201551.921967:0:409198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dae0. 00010000:00000010:1.1:1788201551.922111:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201551.922115:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b4a0. 00010000:00000010:0.1:1788201551.935856:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:0.1:1788201551.935865:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db760. 00010000:00000010:0.1:1788201551.935871:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201551.935876:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d540. 00010000:00000010:0.1:1788201551.951125:0:410562:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201551.951137:0:410562:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db40. 00010000:00000010:1.1:1788201551.954859:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201551.954872:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bb20. 00010000:00000010:0.1:1788201551.958580:0:410562:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00010000:00000010:1.1:1788201551.965128:0:410563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:1.1:1788201551.965133:0:410563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b700. 00010000:00000010:1.1:1788201551.965141:0:410563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:1.1:1788201551.982111:0:410563:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00000080:00000010:1.1:1788201551.982118:0:410563:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b920. 00010000:00000010:1.1:1788201552.048151:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201552.048546:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bf60. 00010000:00000010:0.1:1788201552.049118:0:410564:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201552.049126:0:410564:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d3c0. 00010000:00000010:1.1:1788201552.050584:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646200. 00010000:00000010:1.1:1788201552.058560:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00010000:00000010:0.1:1788201552.071636:0:410567:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:1.1:1788201552.072117:0:409198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00000080:00000010:1.1:1788201552.072124:0:409198:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b4a0. 00000080:00000010:0.1:1788201552.094423:0:410567:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db40. 00010000:00000010:0.1:1788201552.127475:0:410577:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00010000:00000010:1.1:1788201552.134901:0:410576:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:1.1:1788201552.134908:0:410576:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00010000:00000010:2.1:1788201552.153115:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ea00. 00000080:00000010:2.1:1788201552.153123:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bf60. 00000080:00000010:1.1:1788201552.153923:0:410574:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bb20. 00010000:00000010:2.1:1788201552.159148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201552.159162:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db420. 00010000:00000010:1.1:1788201552.184919:0:410574:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00000080:00000010:1.1:1788201552.184927:0:410574:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bfe0. 00010000:00000010:0.1:1788201552.189922:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201552.189938:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d3c0. 00010000:00000010:2.1:1788201552.208666:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:1.1:1788201552.208909:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201552.208914:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b2a0. 00010000:00000010:1.1:1788201552.208920:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:1.1:1788201552.221117:0:410574:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201552.221124:0:410574:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b920. 00010000:00000010:0.1:1788201552.241002:0:410573:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201552.259927:0:410566:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d240. 00010000:00000010:1.1:1788201552.264291:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:1.1:1788201552.264296:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201552.264302:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b3e0. 00010000:00000010:0.1:1788201552.297647:0:410588:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00010000:00000010:1.1:1788201552.313010:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201552.313017:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54be40. 00010000:00000010:1.1:1788201552.313033:0:20:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201552.340860:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b3e0. 00010000:00000010:1.1:1788201552.340873:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:2.1:1788201552.362862:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647000. 00010000:00000010:0.1:1788201552.382894:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201552.382904:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d540. 00010000:00000010:0.1:1788201552.396112:0:410595:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201552.396120:0:410595:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d3c0. 00010000:00000010:1.1:1788201552.396587:0:410594:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201552.396593:0:410594:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b3e0. 00010000:00000010:2.1:1788201552.396948:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201552.396953:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db540. 00010000:00000010:2.1:1788201552.400132:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ec00. 00010000:00000010:2.1:1788201552.400137:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201552.400144:0:409396:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbe80. 00010000:00000010:0.1:1788201552.401713:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:2.1:1788201552.421902:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b800. 00010000:00000010:1.1:1788201552.424114:0:410602:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201552.424119:0:410602:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54be40. 00010000:00000010:2.1:1788201552.424884:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:2.1:1788201552.457235:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:0.1:1788201552.471054:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:1.1:1788201552.478698:0:410608:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:0.1:1788201552.480864:0:410609:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d540. 00010000:00000010:2.1:1788201552.495082:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:2.1:1788201552.495113:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbcc0. 00010000:00000010:2.1:1788201552.495132:0:409392:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201552.495136:0:409392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db980. 00010000:00000010:0.1:1788201552.495859:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201552.495865:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de20. 00010000:00000010:0.1:1788201552.519287:0:410606:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00010000:00000010:2.1:1788201552.520759:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201552.520773:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db7a0. 00010000:00000010:1.1:1788201552.522937:0:410610:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201552.522943:0:410610:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bdc0. 00010000:00000010:2.1:1788201552.526667:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00010000:00000010:2.1:1788201552.528358:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201552.528369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db4c0. 00010000:00000010:1.1:1788201552.530553:0:410601:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201552.530567:0:410601:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b380. 00010000:00000010:1.1:1788201552.587354:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00010000:00000010:0.1:1788201552.605224:0:410613:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201552.605232:0:410613:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3df80. 00010000:00000010:1.1:1788201552.612388:0:410617:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201552.612402:0:410617:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b920. 00010000:00000010:2.1:1788201552.614483:0:410612:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201552.614491:0:410612:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db9e0. 00010000:00000010:0.1:1788201552.619270:0:410618:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201552.619278:0:410618:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3da60. 00010000:00000010:1.1:1788201552.625718:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201552.625727:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bdc0. 00010000:00000010:1.1:1788201552.639281:0:410620:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201552.639294:0:410620:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b360. 00010000:00000010:0.1:1788201552.647122:0:410611:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:2.1:1788201552.648354:0:410622:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:2.1:1788201552.648361:0:410622:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db120. 00010000:00000010:1.1:1788201552.677114:0:410572:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201552.677125:0:410572:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b380. 00010000:00000010:1.1:1788201552.683115:0:410621:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:2.1:1788201552.684856:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00010000:00000010:0.1:1788201552.723118:0:410628:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:0.1:1788201552.723125:0:410628:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db00. 00010000:00000010:0.1:1788201552.723136:0:410628:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00010000:00000010:0.1:1788201552.735730:0:410630:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201552.735735:0:410630:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d240. 00010000:00000010:0.1:1788201552.754131:0:410630:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00010000:00000010:2.1:1788201552.754384:0:409396:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201552.754392:0:409396:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db960. 00010000:00000010:1.1:1788201552.766999:0:410633:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201552.767011:0:410633:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bd80. 00010000:00000010:0.1:1788201552.767840:0:410630:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00000080:00000010:0.1:1788201552.767848:0:410630:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d420. 00010000:00000010:1.1:1788201552.775686:0:410623:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201552.775699:0:410623:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b000. 00010000:00000010:2.1:1788201552.776528:0:410617:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201552.776536:0:410617:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db2e0. 00010000:00000010:2.1:1788201552.776548:0:410617:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:0.1:1788201552.776681:0:410620:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201552.776688:0:410620:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d240. 00010000:00000010:0.1:1788201552.776708:0:410620:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:0.1:1788201552.776711:0:410620:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d7c0. 00010000:00000010:0.1:1788201552.829711:0:410628:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00010000:00000010:0.1:1788201552.829716:0:410628:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201552.829719:0:410628:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dbe0. 00010000:00000010:1.1:1788201552.840805:0:409198:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00010000:00000010:2.1:1788201552.856120:0:410644:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:2.1:1788201552.856130:0:410644:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db2e0. 00010000:00000010:0.1:1788201552.873854:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:1.1:1788201552.902248:0:410643:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201552.902256:0:410643:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bf40. 00010000:00000010:0.1:1788201552.914108:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201552.914113:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dbe0. 00010000:00000010:0.1:1788201552.914230:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201552.914234:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d420. 00010000:00000010:0.1:1788201552.914239:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:0.1:1788201552.914242:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d6c0. 00010000:00000010:1.1:1788201552.921643:0:410643:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201552.921651:0:410643:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b6c0. 00010000:00000010:0.1:1788201552.922220:0:410635:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00010000:00000010:1.1:1788201552.929493:0:410643:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:0.1:1788201552.930109:0:410635:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201552.930118:0:410635:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d240. 00010000:00000010:0.1:1788201552.967738:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:0.1:1788201552.997594:0:410653:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:0.1:1788201552.997612:0:410653:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de20. 00010000:00000010:1.1:1788201553.011455:0:410590:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201553.011461:0:410590:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b900. 00010000:00000010:0.1:1788201553.016139:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:0.1:1788201553.016144:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d420. 00010000:00000010:0.1:1788201553.016149:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00000080:00000010:0.1:1788201553.037111:0:410655:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d6c0. 00010000:00000010:0.1:1788201553.037122:0:410655:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201553.037127:0:410655:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d3c0. 00010000:00000010:0.1:1788201553.057113:0:410667:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00010000:00000010:0.1:1788201553.072400:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:2.1:1788201553.087287:0:410665:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646000. 00000080:00000010:2.1:1788201553.087295:0:410665:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab539cd80. 00010000:00000010:0.1:1788201553.098129:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201553.098142:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d6c0. 00010000:00000010:2.1:1788201553.098226:0:410669:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00010000:00000010:0.1:1788201553.129202:0:410654:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201553.129210:0:410654:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d4c0. 00010000:00000010:0.1:1788201553.138147:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201553.138154:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3da00. 00010000:00000010:1.1:1788201553.150818:0:410674:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00000080:00000010:1.1:1788201553.150826:0:410674:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de80. 00010000:00000010:2.1:1788201553.151662:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201553.151669:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbba0. 00010000:00000010:1.1:1788201553.155327:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00010000:00000010:2.1:1788201553.163823:0:410676:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201553.163837:0:410676:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbe80. 00010000:00000010:0.1:1788201553.176122:0:410681:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00010000:00000010:2.1:1788201553.201251:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:0.1:1788201553.232111:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201553.232121:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd20. 00010000:00000010:1.1:1788201553.232272:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:1.1:1788201553.232279:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bde0. 00010000:00000010:1.1:1788201553.232422:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:1.1:1788201553.232428:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b240. 00010000:00000010:0.1:1788201553.242558:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00000080:00000010:0.1:1788201553.242569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d600. 00010000:00000010:2.1:1788201553.243160:0:410685:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201553.243168:0:410685:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db420. 00010000:00000010:1.1:1788201553.251138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00010000:00000010:0.1:1788201553.252856:0:12:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:2.1:1788201553.270360:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201553.270786:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbba0. 00010000:00000010:1.1:1788201553.298124:0:410695:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:1.1:1788201553.298347:0:410695:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bb60. 00010000:00000010:2.1:1788201553.298817:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00010000:00000010:0.1:1788201553.311264:0:410691:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201553.311271:0:410691:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d420. 00010000:00000010:2.1:1788201553.334497:0:410700:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201553.358034:0:410705:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb40. 00010000:00000010:0.1:1788201553.359703:0:410653:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e600. 00000080:00000010:0.1:1788201553.359715:0:410653:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b200. 00010000:00000010:2.1:1788201553.390244:0:410697:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201553.390260:0:410697:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db8c0. 00010000:00000010:0.1:1788201553.405654:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:0.1:1788201553.405665:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201553.405670:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db40. 00010000:00000010:2.1:1788201553.408865:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00010000:00000010:0.1:1788201553.428656:0:410701:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:0.1:1788201553.428665:0:410701:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201553.428670:0:410701:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d0e0. 00010000:00000010:0.1:1788201553.455118:0:410709:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201553.455134:0:410709:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d4e0. 00010000:00000010:0.1:1788201553.489116:0:410714:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00010000:00000010:0.1:1788201553.489126:0:410714:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:0.1:1788201553.489131:0:410714:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db40. 00010000:00000010:0.1:1788201553.566269:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201553.566333:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d4e0. 00010000:00000010:0.1:1788201553.660272:0:410727:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:0.1:1788201553.660278:0:410727:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3df80. 00010000:00000010:0.1:1788201553.681633:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00010000:00000010:0.1:1788201553.681651:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201553.681655:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db40. 00010000:00000010:0.1:1788201553.724115:0:410745:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00010000:00000010:2.1:1788201553.727299:0:409289:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e200. 00000080:00000010:2.1:1788201553.727311:0:409289:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54ba20. 00010000:00000010:0.1:1788201553.858414:0:410659:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201553.858421:0:410659:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d4c0. 00010000:00000010:2.1:1788201553.916200:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:2.1:1788201553.916214:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db8e0. 00010000:00000010:0.1:1788201553.922249:0:408285:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201553.922262:0:408285:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d960. 00010000:00000010:2.1:1788201553.952326:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201553.952338:0:409408:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db920. 00010000:00000010:0.1:1788201553.961246:0:410775:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00010000:00000010:2.1:1788201553.975300:0:409408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00010000:00000010:0.1:1788201554.001522:0:410787:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:0.1:1788201554.001530:0:410787:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d940. 00010000:00000010:0.1:1788201554.011188:0:410787:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00000080:00000010:0.1:1788201554.011201:0:410787:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d120. 00010000:00000010:0.1:1788201554.043432:0:410651:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00010000:00000010:2.1:1788201554.076650:0:409289:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fa00. 00000080:00000010:2.1:1788201554.076662:0:409289:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b800. 00010000:00000010:2.1:1788201554.095115:0:410806:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201554.095131:0:410806:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db7e0. 00010000:00000010:2.1:1788201554.098704:0:410806:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:2.1:1788201554.126383:0:409289:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646e00. 00000080:00000010:2.1:1788201554.126395:0:409289:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab539c9e0. 00010000:00000010:2.1:1788201554.170394:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:2.1:1788201554.179162:0:410811:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201554.193432:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd20. 00010000:00000010:2.1:1788201554.193459:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:2.1:1788201554.193463:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db9e0. 00010000:00000010:2.1:1788201554.216496:0:410811:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:2.1:1788201554.272028:0:410824:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201554.272043:0:410824:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db9e0. 00010000:00000010:2.1:1788201554.290006:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201554.290018:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db780. 00010000:00000010:2.1:1788201554.336911:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00010000:00000010:2.1:1788201554.369453:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201554.369466:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db800. 00010000:00000010:2.1:1788201554.402849:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:2.1:1788201554.402861:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db920. 00010000:00000010:2.1:1788201554.435066:0:410704:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00010000:00000010:2.1:1788201554.691127:0:410891:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646600. 00000080:00000010:2.1:1788201554.691143:0:410891:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab539c120. 00010000:00000010:2.1:1788201554.708946:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201554.708954:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db640. 00010000:00000010:1.1:1788201554.725260:0:410902:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:2.1:1788201555.123189:0:410966:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647400. 00010000:00000010:2.1:1788201555.188123:0:409438:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:2.1:1788201555.188135:0:409438:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db000. 00010000:00000010:2.1:1788201555.212473:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:2.1:1788201555.212486:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db340. 00010000:00000010:1.1:1788201560.446457:0:411377:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f000. 00000080:00000010:1.1:1788201594.513868:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eaca40. 00000080:00000010:1.1:1788201594.513932:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d6e0. 00010000:00000010:0.1:1788201594.514137:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.514145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00010000:00000010:0.1:1788201594.514149:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.514152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:0.1:1788201594.514158:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d4b9c0. 00000080:00000010:0.1:1788201594.514163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbc00. 00000080:00000010:2.1:1788201594.514244:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e88840. 00000080:00000010:2.1:1788201594.514252:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc440. 00010000:00000010:0.1:1788201594.520978:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737db00. 00010000:00000010:0.1:1788201594.520984:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:0.1:1788201594.520988:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201594.520991:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00000080:00000010:1.1:1788201594.527389:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4398c0. 00000080:00000010:1.1:1788201594.527398:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d960. 00000080:00000010:1.1:1788201594.527411:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d20000. 00000080:00000010:1.1:1788201594.527415:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de20. 00000080:00000010:2.1:1788201594.536132:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab67dac0. 00000080:00000010:2.1:1788201594.536142:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db720. 00000080:00000010:2.1:1788201594.536151:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ed0840. 00000080:00000010:2.1:1788201594.536156:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3da40. 00010000:00000010:1.1:1788201594.536195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201594.536200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:1.1:1788201594.536203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201594.536206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00000080:00000010:1.1:1788201594.536210:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ed6b40. 00000080:00000010:1.1:1788201594.536214:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db560. 00010000:00000010:0.1:1788201594.536386:0:412346:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201594.536391:0:412346:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201594.539207:0:412346:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab47b180. 00000080:00000010:2.1:1788201594.539221:0:412346:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de60. 00000080:00000010:0.1:1788201594.549112:0:412347:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ec6b40. 00000080:00000010:0.1:1788201594.549124:0:412347:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf80. 00000080:00000010:0.1:1788201594.551200:0:1353:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e90000. 00000080:00000010:0.1:1788201594.551210:0:1353:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc3a0. 00000080:00000010:1.1:1788201594.552113:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d4ca40. 00000080:00000010:1.1:1788201594.552120:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d4c0. 00000080:00000010:1.1:1788201594.552127:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab450000. 00000080:00000010:1.1:1788201594.552132:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbe00. 00010000:00000010:1.1:1788201594.557541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201594.557551:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00010000:00000010:2.1:1788201594.561028:0:412349:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201594.561043:0:412349:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1000. 00010000:00000010:2.1:1788201594.561048:0:412349:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.561051:0:412349:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00010000:00000010:1.1:1788201594.564541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201594.564557:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00010000:00000010:1.1:1788201594.569203:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cf00. 00010000:00000010:1.1:1788201594.569209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00010000:00000010:1.1:1788201594.578320:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201594.578330:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00000080:00000010:1.1:1788201594.578340:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ebc200. 00000080:00000010:1.1:1788201594.578348:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db900. 00010000:00000010:0.1:1788201594.582114:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00000080:00000010:1.1:1788201594.582117:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d25280. 00010000:00000010:0.1:1788201594.582119:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00000080:00000010:1.1:1788201594.582126:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dba60. 00000080:00000010:1.1:1788201594.582133:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d318c0. 00000080:00000010:1.1:1788201594.583032:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db560. 00000080:00000010:1.1:1788201594.583040:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ec7380. 00000080:00000010:1.1:1788201594.583049:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d260. 00000080:00000010:2.1:1788201594.588224:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4798c0. 00000080:00000010:2.1:1788201594.588234:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcca0. 00000080:00000010:0.1:1788201594.599118:0:412062:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e8c200. 00000080:00000010:0.1:1788201594.599128:0:412062:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db20. 00010000:00000010:2.1:1788201594.599643:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.599648:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:1.1:1788201594.604014:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab452940. 00000080:00000010:1.1:1788201594.604023:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db3a0. 00010000:00000010:0.1:1788201594.610183:0:412346:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201594.610190:0:412346:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00010000:00000010:0.1:1788201594.610193:0:412346:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737db00. 00010000:00000010:0.1:1788201594.610196:0:412346:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:1.1:1788201594.615171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201594.615183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1000. 00010000:00000010:1.1:1788201594.615228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cf00. 00010000:00000010:1.1:1788201594.615232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00000080:00000010:1.1:1788201594.615352:0:20:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ec9080. 00000080:00000010:1.1:1788201594.615365:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db340. 00010000:00000010:2.1:1788201594.615427:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858300. 00010000:00000010:2.1:1788201594.615433:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646000. 00000080:00000010:2.1:1788201594.615439:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab45f380. 00000080:00000010:2.1:1788201594.615443:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fca20. 00010000:00000010:0.1:1788201594.615698:0:412227:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.615704:0:412227:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0800. 00010000:00000010:0.1:1788201594.615707:0:412227:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.615709:0:412227:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00000080:00000010:0.1:1788201594.615806:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab454200. 00000080:00000010:0.1:1788201594.615810:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db560. 00010000:00000010:2.1:1788201594.623639:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.623645:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00000080:00000010:1.1:1788201594.624412:0:412034:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d339c0. 00000080:00000010:1.1:1788201594.624503:0:412034:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbaa0. 00000080:00000010:2.1:1788201594.636251:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d0b180. 00000080:00000010:2.1:1788201594.636266:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcb00. 00010000:00000010:1.1:1788201594.652116:0:412034:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cf00. 00010000:00000010:1.1:1788201594.652122:0:412034:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00000080:00000010:1.1:1788201594.652130:0:412034:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d22940. 00000080:00000010:1.1:1788201594.652213:0:412034:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db540. 00000080:00000010:0.1:1788201594.652456:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5edeb40. 00000080:00000010:0.1:1788201594.652462:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbec0. 00010000:00000010:0.1:1788201594.652469:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.652474:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34200. 00010000:00000010:0.1:1788201594.652478:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.652481:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647e00. 00010000:00000010:1.1:1788201594.662115:0:412062:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201594.662122:0:412062:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201594.671345:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eaeb40. 00000080:00000010:1.1:1788201594.671353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db3a0. 00000080:00000010:1.1:1788201594.671361:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d57380. 00000080:00000010:1.1:1788201594.671365:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dba60. 00010000:00000010:1.1:1788201594.671370:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201594.671373:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:0.1:1788201594.671432:0:412358:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737db00. 00010000:00000010:0.1:1788201594.671439:0:412358:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00000080:00000010:0.1:1788201594.671445:0:412358:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab468000. 00000080:00000010:0.1:1788201594.671450:0:412358:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db20. 00000080:00000010:0.1:1788201594.671456:0:412358:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab67a940. 00000080:00000010:0.1:1788201594.671461:0:412358:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441640. 00010000:00000010:2.1:1788201594.671678:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.671684:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:2.1:1788201594.671687:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201594.671689:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00000080:00000010:2.1:1788201594.671694:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e918c0. 00000080:00000010:2.1:1788201594.671698:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa4417a0. 00010000:00000010:0.1:1788201594.675675:0:412356:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.675682:0:412356:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00000080:00000010:0.1:1788201594.675688:0:412356:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eda940. 00000080:00000010:0.1:1788201594.675694:0:412356:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3da40. 00010000:00000010:0.1:1788201594.675699:0:412356:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.675702:0:412356:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:1.1:1788201594.676252:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cf00. 00010000:00000010:1.1:1788201594.676259:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1000. 00010000:00000010:1.1:1788201594.676262:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.676265:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00000080:00000010:1.1:1788201594.676334:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d27380. 00000080:00000010:1.1:1788201594.676343:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db340. 00010000:00000010:2.1:1788201594.681106:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201594.681111:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:1.1:1788201594.681216:0:412354:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fca20. 00000080:00000010:1.1:1788201594.688882:0:412062:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab46f380. 00000080:00000010:1.1:1788201594.688898:0:412062:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d880. 00010000:00000010:1.1:1788201594.693302:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.693309:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00010000:00000010:0.1:1788201594.693668:0:412346:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.693675:0:412346:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34200. 00000080:00000010:0.1:1788201594.693691:0:412346:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e998c0. 00000080:00000010:0.1:1788201594.693698:0:412346:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3df60. 00000080:00000010:1.1:1788201594.700662:0:412360:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d56300. 00000080:00000010:1.1:1788201594.700673:0:412360:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db520. 00000080:00000010:0.1:1788201594.701144:0:412361:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ed98c0. 00000080:00000010:0.1:1788201594.701151:0:412361:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbc60. 00000080:00000010:2.1:1788201594.704949:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ea0840. 00000080:00000010:2.1:1788201594.704960:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc1a0. 00010000:00000010:0.1:1788201594.717370:0:412362:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.717376:0:412362:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:1.1:1788201594.718889:0:412062:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201594.718895:0:412062:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00010000:00000010:2.1:1788201594.723632:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858700. 00010000:00000010:2.1:1788201594.723646:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646a00. 00000080:00000010:2.1:1788201594.723653:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eee300. 00000080:00000010:2.1:1788201594.723661:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa4414e0. 00010000:00000010:0.1:1788201594.737285:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.737290:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00000080:00000010:0.1:1788201594.737298:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5edf380. 00000080:00000010:0.1:1788201594.737305:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d340. 00010000:00000010:1.1:1788201594.740573:0:412062:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201594.740580:0:412062:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1000. 00010000:00000010:2.1:1788201594.746654:0:412231:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201594.746661:0:412231:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:2.1:1788201594.746670:0:412231:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab469080. 00000080:00000010:2.1:1788201594.746678:0:412231:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db340. 00010000:00000010:1.1:1788201594.747127:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.747194:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646e00. 00000080:00000010:0.1:1788201594.749173:0:412361:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5e98000. 00000080:00000010:0.1:1788201594.749197:0:412361:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d9c0. 00000080:00000010:0.1:1788201594.756356:0:412359:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ed8000. 00000080:00000010:0.1:1788201594.756366:0:412359:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d400. 00010000:00000010:2.1:1788201594.757707:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.757721:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:1.1:1788201594.757770:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d53180. 00000080:00000010:1.1:1788201594.757780:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dba60. 00010000:00000010:2.1:1788201594.762547:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201594.762555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:2.1:1788201594.762562:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5efb180. 00000080:00000010:2.1:1788201594.762569:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcba0. 00000080:00000010:1.1:1788201594.763151:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d42100. 00000080:00000010:1.1:1788201594.763158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441120. 00010000:00000010:1.1:1788201594.763167:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.763171:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646200. 00010000:00000010:0.1:1788201594.763298:0:412062:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.763304:0:412062:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34200. 00000080:00000010:1.1:1788201594.773317:0:412345:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee7380. 00000080:00000010:1.1:1788201594.773325:0:412345:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d9a0. 00000080:00000010:2.1:1788201594.779220:0:412034:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab46b9c0. 00000080:00000010:2.1:1788201594.779229:0:412034:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc700. 00010000:00000010:0.1:1788201594.786116:0:13:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.786132:0:13:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1000. 00010000:00000010:2.1:1788201594.787319:0:412368:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201594.787332:0:412368:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:2.1:1788201594.790328:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.790334:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00010000:00000010:1.1:1788201594.801114:0:412227:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.801120:0:412227:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:0.1:1788201594.804391:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ea2940. 00000080:00000010:0.1:1788201594.804398:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3daa0. 00000080:00000010:2.1:1788201594.807329:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab408000. 00000080:00000010:2.1:1788201594.807337:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc8a0. 00000080:00000010:2.1:1788201594.807343:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d45ac0. 00000080:00000010:2.1:1788201594.807347:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf20. 00000080:00000010:0.1:1788201594.809864:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d51080. 00000080:00000010:0.1:1788201594.809892:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0e0. 00000080:00000010:1.1:1788201594.811064:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5efe300. 00000080:00000010:1.1:1788201594.811073:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dba60. 00010000:00000010:0.1:1788201594.817187:0:412364:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.817199:0:412364:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646400. 00000080:00000010:0.1:1788201594.817206:0:412364:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4718c0. 00000080:00000010:0.1:1788201594.817213:0:412364:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc9c0. 00010000:00000010:0.1:1788201594.827896:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.827903:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34200. 00010000:00000010:0.1:1788201594.838222:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.838231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:0.1:1788201594.838236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737db00. 00010000:00000010:0.1:1788201594.838238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00000080:00000010:0.1:1788201594.838245:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ea98c0. 00000080:00000010:0.1:1788201594.838256:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441340. 00010000:00000010:2.1:1788201594.838311:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.838316:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:2.1:1788201594.838323:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eeb9c0. 00000080:00000010:2.1:1788201594.838328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa4416e0. 00000080:00000010:2.1:1788201594.838335:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d10000. 00000080:00000010:2.1:1788201594.838385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441ae0. 00000080:00000010:0.1:1788201594.842338:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ea1080. 00000080:00000010:0.1:1788201594.842356:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3da40. 00010000:00000010:1.1:1788201594.845195:0:391368:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.845212:0:391368:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:0.1:1788201594.846119:0:412369:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d46b40. 00000080:00000010:0.1:1788201594.846252:0:412369:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc680. 00000080:00000010:1.1:1788201594.873349:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab40ca40. 00000080:00000010:1.1:1788201594.873360:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db940. 00000080:00000010:0.1:1788201594.875410:0:412369:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab474a40. 00000080:00000010:0.1:1788201594.875423:0:412369:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3daa0. 00010000:00000010:1.1:1788201594.887345:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.887352:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35e00. 00010000:00000010:1.1:1788201594.887358:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201594.887362:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34200. 00010000:00000010:2.1:1788201594.887448:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.887454:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:2.1:1788201594.887457:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201594.887459:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:0.1:1788201594.914126:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737db00. 00010000:00000010:0.1:1788201594.914132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34e00. 00000080:00000010:0.1:1788201594.923149:0:411067:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d45280. 00000080:00000010:0.1:1788201594.923157:0:411067:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d400. 00010000:00000010:0.1:1788201594.923165:0:411067:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.923172:0:411067:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00010000:00000010:0.1:1788201594.923176:0:411067:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.923179:0:411067:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:1.1:1788201594.927566:0:412376:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201594.927574:0:412376:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201594.927582:0:412376:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ea2100. 00000080:00000010:1.1:1788201594.927600:0:412376:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db20. 00000080:00000010:1.1:1788201594.929952:0:391368:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eb0840. 00000080:00000010:1.1:1788201594.929969:0:391368:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc60. 00010000:00000010:1.1:1788201594.929978:0:391368:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201594.929993:0:391368:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:1.1:1788201594.929998:0:391368:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.930001:0:391368:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00010000:00000010:2.1:1788201594.930383:0:412377:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201594.930390:0:412377:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646600. 00000080:00000010:2.1:1788201594.930394:0:412377:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d0d280. 00000080:00000010:2.1:1788201594.930400:0:412377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc8a0. 00000080:00000010:2.1:1788201594.930408:0:412377:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eba100. 00000080:00000010:2.1:1788201594.930412:0:412377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc040. 00000080:00000010:2.1:1788201594.930418:0:412377:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ef4a40. 00000080:00000010:2.1:1788201594.930422:0:412377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db820. 00000080:00000010:2.1:1788201594.930429:0:412377:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5efeb40. 00000080:00000010:2.1:1788201594.930435:0:412377:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441840. 00010000:00000010:2.1:1788201594.940223:0:412231:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201594.940230:0:412231:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:2.1:1788201594.940234:0:412231:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201594.940238:0:412231:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:0.1:1788201594.940349:0:412381:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab475ac0. 00000080:00000010:0.1:1788201594.940356:0:412381:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de60. 00010000:00000010:2.1:1788201594.948235:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201594.948241:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:2.1:1788201594.962226:0:412385:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201594.962234:0:412385:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201594.962242:0:412385:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee8840. 00000080:00000010:2.1:1788201594.962251:0:412385:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc940. 00000080:00000010:1.1:1788201594.962270:0:412382:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ec4200. 00000080:00000010:1.1:1788201594.962279:0:412382:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db940. 00000080:00000010:0.1:1788201594.963221:0:412384:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d47380. 00000080:00000010:0.1:1788201594.963230:0:412384:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db40. 00000080:00000010:1.1:1788201594.967295:0:19:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eb3180. 00000080:00000010:1.1:1788201594.967323:0:19:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db3a0. 00010000:00000010:1.1:1788201594.967336:0:19:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.967344:0:19:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00000080:00000010:2.1:1788201594.967502:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d0f380. 00000080:00000010:2.1:1788201594.967508:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc380. 00000080:00000010:2.1:1788201594.967515:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eff380. 00000080:00000010:2.1:1788201594.967529:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc220. 00010000:00000010:2.1:1788201594.967534:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201594.967537:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:0.1:1788201594.970114:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.970121:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:0.1:1788201594.970125:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.970128:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00010000:00000010:1.1:1788201594.982588:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201594.982596:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00000080:00000010:0.1:1788201594.983143:0:412386:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab45d280. 00000080:00000010:0.1:1788201594.983150:0:412386:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d400. 00010000:00000010:2.1:1788201594.983716:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201594.983723:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:0.1:1788201594.988202:0:412386:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d43180. 00000080:00000010:0.1:1788201594.988211:0:412386:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d280. 00000080:00000010:0.1:1788201594.988217:0:412386:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eb2940. 00000080:00000010:0.1:1788201594.988235:0:412386:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbaa0. 00010000:00000010:2.1:1788201594.988648:0:412154:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201594.988653:0:412154:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201594.988658:0:412154:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab405ac0. 00000080:00000010:2.1:1788201594.988662:0:412154:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc5c0. 00000080:00000010:1.1:1788201594.992434:0:412380:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eca940. 00000080:00000010:1.1:1788201594.992452:0:412380:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db540. 00000080:00000010:2.1:1788201594.995215:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d12940. 00000080:00000010:2.1:1788201594.995226:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc680. 00010000:00000010:0.1:1788201594.997445:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201594.997452:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00010000:00000010:0.1:1788201594.997455:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201594.997458:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:0.1:1788201595.017113:0:412034:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.017120:0:412034:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34e00. 00000080:00000010:0.1:1788201595.017448:0:412034:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d41080. 00000080:00000010:0.1:1788201595.017457:0:412034:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de60. 00000080:00000010:0.1:1788201595.017466:0:412034:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab47a100. 00000080:00000010:0.1:1788201595.017472:0:412034:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d060. 00010000:00000010:1.1:1788201595.026132:0:412386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.026141:0:412386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:1.1:1788201595.026177:0:412386:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.026181:0:412386:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201595.026598:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab41dac0. 00000080:00000010:2.1:1788201595.026607:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc8a0. 00010000:00000010:2.1:1788201595.026617:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201595.026623:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:2.1:1788201595.035147:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ef18c0. 00000080:00000010:2.1:1788201595.035154:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fca40. 00010000:00000010:0.1:1788201595.035212:0:412034:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:0.1:1788201595.035218:0:412034:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:0.1:1788201595.035224:0:412034:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eceb40. 00000080:00000010:0.1:1788201595.035228:0:412034:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d7a0. 00000080:00000010:0.1:1788201595.035234:0:412034:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ed39c0. 00000080:00000010:0.1:1788201595.035238:0:412034:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbe60. 00010000:00000010:0.1:1788201595.035245:0:412034:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737db00. 00010000:00000010:0.1:1788201595.035248:0:412034:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00010000:00000010:0.1:1788201595.035251:0:412034:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.035253:0:412034:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00000080:00000010:1.1:1788201595.048124:0:412397:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab414200. 00000080:00000010:1.1:1788201595.048149:0:412397:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc680. 00010000:00000010:0.1:1788201595.048223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201595.048231:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00010000:00000010:2.1:1788201595.050107:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201595.050113:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:0.1:1788201595.060117:0:412399:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d09080. 00000080:00000010:0.1:1788201595.060127:0:412399:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d280. 00010000:00000010:0.1:1788201595.060133:0:412399:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:0.1:1788201595.060137:0:412399:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00000080:00000010:0.1:1788201595.060142:0:412399:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d44200. 00000080:00000010:0.1:1788201595.060147:0:412399:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d560. 00000080:00000010:1.1:1788201595.072918:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ec6300. 00000080:00000010:1.1:1788201595.072926:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db80. 00010000:00000010:2.1:1788201595.075314:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201595.075322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00010000:00000010:0.1:1788201595.078174:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201595.078188:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34e00. 00000080:00000010:0.1:1788201595.078196:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab414a40. 00000080:00000010:0.1:1788201595.078203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dba60. 00000080:00000010:2.1:1788201595.078504:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab402940. 00000080:00000010:2.1:1788201595.078511:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc2c0. 00000080:00000010:2.1:1788201595.078518:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ed2100. 00000080:00000010:2.1:1788201595.078522:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd60. 00000080:00000010:0.1:1788201595.078543:0:412392:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d1dac0. 00000080:00000010:0.1:1788201595.078547:0:412392:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441f60. 00010000:00000010:2.1:1788201595.084084:0:412375:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201595.084106:0:412375:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:2.1:1788201595.084111:0:412375:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.084114:0:412375:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00010000:00000010:1.1:1788201595.097885:0:412394:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.097896:0:412394:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:0.1:1788201595.104115:0:412390:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201595.104126:0:412390:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201595.104134:0:412390:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d098c0. 00000080:00000010:0.1:1788201595.104142:0:412390:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d060. 00000080:00000010:2.1:1788201595.105670:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab40f380. 00000080:00000010:2.1:1788201595.105681:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc500. 00000080:00000010:2.1:1788201595.105689:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ecdac0. 00000080:00000010:2.1:1788201595.105694:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc620. 00010000:00000010:1.1:1788201595.107678:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.107685:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00000080:00000010:0.1:1788201595.111378:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ecb180. 00000080:00000010:0.1:1788201595.111403:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0a0. 00010000:00000010:2.1:1788201595.126555:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.126563:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00010000:00000010:1.1:1788201595.126814:0:412405:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.126821:0:412405:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34e00. 00000080:00000010:1.1:1788201595.126831:0:412405:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d32100. 00000080:00000010:1.1:1788201595.126837:0:412405:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d180. 00000080:00000010:0.1:1788201595.132894:0:1353:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab433180. 00000080:00000010:0.1:1788201595.132906:0:1353:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d080. 00010000:00000010:2.1:1788201595.133277:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201595.133283:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:1.1:1788201595.142429:0:412407:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d118c0. 00000080:00000010:1.1:1788201595.142465:0:20:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d120. 00000080:00000010:2.1:1788201595.151216:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab409080. 00000080:00000010:2.1:1788201595.151224:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc8c0. 00010000:00000010:1.1:1788201595.152504:0:412407:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.152510:0:412407:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:0.1:1788201595.155214:0:412408:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201595.155219:0:412408:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:2.1:1788201595.155359:0:256128:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5edca40. 00000080:00000010:2.1:1788201595.155366:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc2c0. 00000080:00000010:0.1:1788201595.163855:0:412406:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab42c200. 00000080:00000010:0.1:1788201595.163879:0:412406:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d560. 00010000:00000010:1.1:1788201595.164845:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.164855:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201595.179741:0:412154:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d30000. 00000080:00000010:1.1:1788201595.179748:0:412154:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db340. 00000080:00000010:1.1:1788201595.179755:0:412154:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d15ac0. 00000080:00000010:1.1:1788201595.179760:0:412154:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db720. 00010000:00000010:2.1:1788201595.181925:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201595.181935:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00000080:00000010:0.1:1788201595.184077:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ed0000. 00000080:00000010:0.1:1788201595.184188:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441240. 00010000:00000010:1.1:1788201595.191510:0:412416:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.191516:0:412416:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35c00. 00010000:00000010:2.1:1788201595.192252:0:412414:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.192260:0:412414:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201595.192266:0:412414:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab41a100. 00000080:00000010:2.1:1788201595.192271:0:412414:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc220. 00000080:00000010:2.1:1788201595.192278:0:412414:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee39c0. 00000080:00000010:2.1:1788201595.192281:0:412414:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc620. 00010000:00000010:1.1:1788201595.192402:0:391368:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.192408:0:391368:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00010000:00000010:1.1:1788201595.192412:0:391368:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.192414:0:391368:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00000080:00000010:0.1:1788201595.225112:0:412417:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d16b40. 00000080:00000010:1.1:1788201595.225537:0:412418:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab42dac0. 00000080:00000010:1.1:1788201595.225544:0:412418:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc20. 00000080:00000010:0.1:1788201595.226009:0:412417:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db9e0. 00010000:00000010:0.1:1788201595.226018:0:412417:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201595.226024:0:412417:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:0.1:1788201595.226027:0:412417:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:0.1:1788201595.226031:0:412417:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00010000:00000010:2.1:1788201595.234554:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.234565:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:1.1:1788201595.238544:0:412231:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d36b40. 00000080:00000010:1.1:1788201595.238563:0:412231:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dba00. 00010000:00000010:1.1:1788201595.257116:0:412154:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.257121:0:412154:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00000080:00000010:2.1:1788201595.257210:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee2100. 00000080:00000010:2.1:1788201595.257216:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcee0. 00010000:00000010:0.1:1788201595.265288:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:0.1:1788201595.265295:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201595.270115:0:412424:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eb9080. 00000080:00000010:0.1:1788201595.270124:0:412424:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d560. 00010000:00000010:2.1:1788201595.271302:0:256128:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.271309:0:256128:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00010000:00000010:1.1:1788201595.281115:0:412231:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00000080:00000010:0.1:1788201595.281119:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d10840. 00010000:00000010:1.1:1788201595.281120:0:412231:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00000080:00000010:1.1:1788201595.281124:0:412231:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab404a40. 00000080:00000010:0.1:1788201595.281127:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dae0. 00000080:00000010:1.1:1788201595.281130:0:412231:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db4e0. 00000080:00000010:0.1:1788201595.281134:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab420000. 00000080:00000010:0.1:1788201595.281138:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcd40. 00010000:00000010:0.1:1788201595.290879:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:0.1:1788201595.290888:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00010000:00000010:0.1:1788201595.290906:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201595.290909:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:2.1:1788201595.298118:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee9080. 00000080:00000010:2.1:1788201595.298198:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc220. 00000080:00000010:1.1:1788201595.298419:0:412154:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d3dac0. 00000080:00000010:1.1:1788201595.298426:0:412154:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db340. 00010000:00000010:1.1:1788201595.298433:0:412154:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.298437:0:412154:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:0.1:1788201595.316353:0:412428:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.316361:0:412428:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201595.316374:0:412428:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eb98c0. 00000080:00000010:0.1:1788201595.316394:0:412428:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d9e0. 00010000:00000010:1.1:1788201595.320506:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.320512:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00010000:00000010:2.1:1788201595.321222:0:412429:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.321228:0:412429:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201595.321234:0:412429:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ede300. 00000080:00000010:2.1:1788201595.321239:0:412429:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcaa0. 00000080:00000010:1.1:1788201595.325118:0:412154:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab435280. 00000080:00000010:1.1:1788201595.325306:0:412154:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db720. 00000080:00000010:0.1:1788201595.325327:0:412424:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d15280. 00000080:00000010:0.1:1788201595.325335:0:412424:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d920. 00000080:00000010:0.1:1788201595.325344:0:412424:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab422940. 00000080:00000010:0.1:1788201595.325347:0:412424:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de40. 00010000:00000010:0.1:1788201595.325353:0:412424:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201595.325359:0:412424:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00010000:00000010:0.1:1788201595.325779:0:412424:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:0.1:1788201595.325785:0:412424:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35c00. 00010000:00000010:0.1:1788201595.332114:0:412428:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00000080:00000010:1.1:1788201595.332114:0:412430:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee5ac0. 00010000:00000010:0.1:1788201595.332128:0:412428:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:1.1:1788201595.332137:0:412430:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d080. 00000080:00000010:1.1:1788201595.332148:0:412430:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d3eb40. 00000080:00000010:1.1:1788201595.332153:0:412430:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dba00. 00010000:00000010:2.1:1788201595.402527:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.402542:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:2.1:1788201595.402554:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab41c200. 00000080:00000010:2.1:1788201595.402560:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d1c0. 00010000:00000010:1.1:1788201595.412113:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.412289:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:1.1:1788201595.412295:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.412297:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00010000:00000010:1.1:1788201595.412300:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.412303:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201595.412307:0:412319:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab440840. 00000080:00000010:1.1:1788201595.412312:0:412319:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db340. 00000080:00000010:0.1:1788201595.412666:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d17380. 00000080:00000010:0.1:1788201595.412672:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dbe0. 00010000:00000010:0.1:1788201595.414853:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.414867:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00010000:00000010:2.1:1788201595.420196:0:412443:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ab00. 00010000:00000010:2.1:1788201595.420203:0:412443:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:0.1:1788201595.420481:0:412444:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ef0840. 00000080:00000010:0.1:1788201595.420489:0:412444:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc180. 00000080:00000010:1.1:1788201595.420540:0:412154:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ee0840. 00000080:00000010:1.1:1788201595.420561:0:412154:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbc60. 00000080:00000010:1.1:1788201595.420569:0:412154:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d38840. 00000080:00000010:1.1:1788201595.420574:0:412154:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db9a0. 00000080:00000010:0.1:1788201595.425520:0:412444:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d16300. 00000080:00000010:0.1:1788201595.425529:0:412444:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d920. 00010000:00000010:2.1:1788201595.426111:0:412443:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.426118:0:412443:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00010000:00000010:1.1:1788201595.428054:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.428070:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:1.1:1788201595.430491:0:412446:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.430504:0:412446:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00000080:00000010:2.1:1788201595.431843:0:412445:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab426300. 00000080:00000010:2.1:1788201595.431857:0:412445:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fce60. 00010000:00000010:1.1:1788201595.437458:0:412446:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.437473:0:412446:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00010000:00000010:2.1:1788201595.464381:0:412448:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.464386:0:412448:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00010000:00000010:0.1:1788201595.473201:0:412437:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.473213:0:412437:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35c00. 00000080:00000010:1.1:1788201595.473214:0:412449:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab454a40. 00000080:00000010:1.1:1788201595.473223:0:412449:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db4e0. 00000080:00000010:1.1:1788201595.473232:0:412449:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d3d280. 00000080:00000010:1.1:1788201595.473236:0:412449:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db960. 00010000:00000010:0.1:1788201595.473263:0:412437:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:0.1:1788201595.473267:0:412437:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201595.473285:0:412437:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5eea940. 00000080:00000010:0.1:1788201595.473304:0:412437:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d460. 00000080:00000010:1.1:1788201595.485347:0:412231:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5edc200. 00000080:00000010:1.1:1788201595.485370:0:412231:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db720. 00010000:00000010:1.1:1788201595.503075:0:412453:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.503083:0:412453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00000080:00000010:2.1:1788201595.503555:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab42a100. 00000080:00000010:2.1:1788201595.503564:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc3a0. 00000080:00000010:0.1:1788201595.504845:0:412454:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d34a40. 00000080:00000010:0.1:1788201595.504855:0:412454:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d000. 00010000:00000010:0.1:1788201595.504873:0:412454:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:0.1:1788201595.504878:0:412454:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00010000:00000010:1.1:1788201595.511114:0:412154:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.511483:0:412154:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00010000:00000010:1.1:1788201595.511487:0:412154:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.511490:0:412154:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:2.1:1788201595.532202:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.532209:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201595.532218:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab450840. 00000080:00000010:2.1:1788201595.532225:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db1c0. 00000080:00000010:1.1:1788201595.532494:0:412458:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ef3180. 00000080:00000010:1.1:1788201595.532500:0:412458:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d3e0. 00010000:00000010:1.1:1788201595.555518:0:412453:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.555527:0:412453:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:1.1:1788201595.555535:0:412453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d28000. 00000080:00000010:1.1:1788201595.555541:0:412453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dba20. 00000080:00000010:1.1:1788201595.559235:0:412453:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d35ac0. 00000080:00000010:1.1:1788201595.559243:0:412453:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d080. 00000080:00000010:0.1:1788201595.559830:0:412319:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ef2940. 00000080:00000010:0.1:1788201595.559838:0:412319:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db400. 00010000:00000010:0.1:1788201595.559951:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:0.1:1788201595.559956:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00000080:00000010:2.1:1788201595.569381:0:412462:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab435ac0. 00000080:00000010:2.1:1788201595.569390:0:412462:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc440. 00010000:00000010:1.1:1788201595.581377:0:412463:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.581385:0:412463:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:0.1:1788201595.581852:0:412460:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.581868:0:412460:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00010000:00000010:2.1:1788201595.586426:0:412464:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.586436:0:412464:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:1.1:1788201595.596844:0:412467:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ef98c0. 00000080:00000010:1.1:1788201595.596869:0:412467:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db480. 00010000:00000010:1.1:1788201595.598066:0:391368:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.598072:0:391368:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00000080:00000010:1.1:1788201595.598078:0:391368:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d2eb40. 00000080:00000010:1.1:1788201595.598086:0:391368:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db680. 00000080:00000010:0.1:1788201595.606195:0:412469:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5ef6300. 00000080:00000010:0.1:1788201595.606219:0:412469:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d920. 00000080:00000010:2.1:1788201595.620924:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab436b40. 00000080:00000010:2.1:1788201595.620939:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc880. 00010000:00000010:1.1:1788201595.623109:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.623114:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00010000:00000010:0.1:1788201595.623227:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.623233:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:1.1:1788201595.623707:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d2c200. 00000080:00000010:1.1:1788201595.623712:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db760. 00010000:00000010:1.1:1788201595.642507:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.642513:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00000080:00000010:0.1:1788201595.646118:0:412471:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5efc200. 00000080:00000010:0.1:1788201595.646127:0:412471:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d680. 00000080:00000010:0.1:1788201595.646139:0:412471:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d28840. 00000080:00000010:0.1:1788201595.646143:0:412471:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf60. 00000080:00000010:1.1:1788201595.646200:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab40d280. 00000080:00000010:1.1:1788201595.646206:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbbe0. 00010000:00000010:1.1:1788201595.646212:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.646216:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00010000:00000010:2.1:1788201595.646891:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.646900:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00010000:00000010:1.1:1788201595.655174:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.655181:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:1.1:1788201595.655184:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ce00. 00010000:00000010:1.1:1788201595.655186:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:2.1:1788201595.655908:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab44a100. 00000080:00000010:2.1:1788201595.655915:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc000. 00000080:00000010:1.1:1788201595.665132:0:412435:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d298c0. 00000080:00000010:1.1:1788201595.665148:0:412435:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db680. 00000080:00000010:1.1:1788201595.690211:0:412476:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab415ac0. 00000080:00000010:1.1:1788201595.690223:0:412476:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db360. 00000080:00000010:1.1:1788201595.690233:0:412476:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab40eb40. 00000080:00000010:1.1:1788201595.690254:0:412476:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d2e0. 00010000:00000010:2.1:1788201595.690575:0:412444:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.690580:0:412444:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00010000:00000010:0.1:1788201595.691057:0:412475:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.691063:0:412475:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201595.716112:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d2d280. 00000080:00000010:0.1:1788201595.716121:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d560. 00000080:00000010:0.1:1788201595.717784:0:412205:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d2ca40. 00000080:00000010:0.1:1788201595.717795:0:412205:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbbe0. 00010000:00000010:1.1:1788201595.718199:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ce00. 00010000:00000010:1.1:1788201595.718205:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00010000:00000010:1.1:1788201595.718208:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.718212:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00010000:00000010:1.1:1788201595.718214:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.718216:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00000080:00000010:1.1:1788201595.724894:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5efa940. 00000080:00000010:1.1:1788201595.724913:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db760. 00010000:00000010:0.1:1788201595.746038:0:412484:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.746045:0:412484:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:1.1:1788201595.746479:0:412319:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5efdac0. 00000080:00000010:1.1:1788201595.746489:0:412319:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb40. 00000080:00000010:2.1:1788201595.754142:0:412481:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d2a100. 00000080:00000010:2.1:1788201595.754150:0:412481:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d120. 00010000:00000010:1.1:1788201595.755481:0:412477:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.755487:0:412477:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00010000:00000010:1.1:1788201595.771258:0:412205:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.771316:0:412205:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00000080:00000010:1.1:1788201595.786956:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab415280. 00000080:00000010:1.1:1788201595.786963:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db360. 00000080:00000010:1.1:1788201595.786969:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab416b40. 00000080:00000010:1.1:1788201595.786973:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbba0. 00010000:00000010:1.1:1788201595.786979:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.786986:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00010000:00000010:1.1:1788201595.811161:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.811170:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201595.817883:0:412491:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d2dac0. 00000080:00000010:1.1:1788201595.817892:0:412491:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc2e0. 00010000:00000010:1.1:1788201595.820867:0:412491:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.820879:0:412491:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00000080:00000010:1.1:1788201595.821018:0:412491:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab411080. 00000080:00000010:1.1:1788201595.821028:0:412491:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd80. 00010000:00000010:2.1:1788201595.828111:0:412444:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.828121:0:412444:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647000. 00000080:00000010:2.1:1788201595.869174:0:412231:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab476300. 00000080:00000010:2.1:1788201595.869182:0:412231:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441c80. 00010000:00000010:1.1:1788201595.877146:0:412497:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.877160:0:412497:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00000080:00000010:1.1:1788201595.877168:0:412497:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab412940. 00000080:00000010:1.1:1788201595.877188:0:412497:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb40. 00010000:00000010:1.1:1788201595.877197:0:412497:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.877200:0:412497:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201595.882718:0:412502:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d32940. 00000080:00000010:1.1:1788201595.882728:0:412502:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbba0. 00000080:00000010:1.1:1788201595.882736:0:412502:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab41f380. 00000080:00000010:1.1:1788201595.882752:0:412502:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db720. 00010000:00000010:1.1:1788201595.882758:0:412502:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ce00. 00010000:00000010:1.1:1788201595.882763:0:412502:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00010000:00000010:2.1:1788201595.910929:0:412231:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.910941:0:412231:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201595.939921:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab444a40. 00000080:00000010:2.1:1788201595.939940:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441aa0. 00000080:00000010:2.1:1788201595.939948:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4598c0. 00000080:00000010:2.1:1788201595.939955:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc000. 00010000:00000010:2.1:1788201595.939963:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201595.939969:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00010000:00000010:2.1:1788201595.939995:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.939999:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201595.940004:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab46ca40. 00000080:00000010:2.1:1788201595.940009:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc620. 00010000:00000010:1.1:1788201595.942905:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ce00. 00010000:00000010:1.1:1788201595.942913:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00000080:00000010:1.1:1788201595.942920:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab425ac0. 00000080:00000010:1.1:1788201595.942926:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db000. 00010000:00000010:1.1:1788201595.942931:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.942934:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00000080:00000010:1.1:1788201595.942938:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d37380. 00000080:00000010:1.1:1788201595.942944:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb80. 00000080:00000010:1.1:1788201595.942949:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab401080. 00000080:00000010:1.1:1788201595.942953:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db480. 00010000:00000010:1.1:1788201595.946557:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201595.946568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00000080:00000010:2.1:1788201595.954121:0:412507:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab47d280. 00000080:00000010:2.1:1788201595.954130:0:412507:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc180. 00010000:00000010:2.1:1788201595.954138:0:412507:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201595.954142:0:412507:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00010000:00000010:1.1:1788201595.957262:0:412503:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201595.957281:0:412503:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:1.1:1788201595.957285:0:412503:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.957288:0:412503:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:0.1:1788201595.958827:0:412319:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab420840. 00000080:00000010:0.1:1788201595.958836:0:412319:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db720. 00000080:00000010:1.1:1788201595.985596:0:412450:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d36300. 00000080:00000010:1.1:1788201595.985605:0:412450:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbba0. 00010000:00000010:0.1:1788201595.992073:0:412512:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201595.992081:0:412512:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00010000:00000010:2.1:1788201595.992729:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201595.992736:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:2.1:1788201595.992743:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab45ca40. 00000080:00000010:2.1:1788201595.992752:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc760. 00000080:00000010:2.1:1788201595.997149:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab47f380. 00000080:00000010:2.1:1788201595.997163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fce60. 00010000:00000010:2.1:1788201595.997172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201595.997178:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00010000:00000010:2.1:1788201595.997183:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201595.997186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:0.1:1788201595.997236:0:412449:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab42d280. 00000080:00000010:0.1:1788201595.997243:0:412449:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dac0. 00010000:00000010:1.1:1788201595.998842:0:412495:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201595.998865:0:412495:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201596.028955:0:412493:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d33180. 00000080:00000010:1.1:1788201596.028962:0:412493:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbc80. 00000080:00000010:2.1:1788201596.029206:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4318c0. 00000080:00000010:2.1:1788201596.029212:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb60. 00010000:00000010:0.1:1788201596.049363:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.049370:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00000080:00000010:2.1:1788201596.066840:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab472100. 00000080:00000010:2.1:1788201596.066864:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc400. 00010000:00000010:2.1:1788201596.066874:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201596.066879:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201596.067111:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab45dac0. 00000080:00000010:2.1:1788201596.067116:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc180. 00010000:00000010:2.1:1788201596.067121:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.067124:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:2.1:1788201596.067130:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4039c0. 00000080:00000010:2.1:1788201596.067149:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc060. 00010000:00000010:2.1:1788201596.086500:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201596.086507:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201596.086516:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d06b40. 00000080:00000010:2.1:1788201596.086524:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc640. 00000080:00000010:0.1:1788201596.088455:0:412515:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab42e300. 00000080:00000010:0.1:1788201596.088465:0:412515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d220. 00010000:00000010:2.1:1788201596.097021:0:412522:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.097029:0:412522:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00000080:00000010:2.1:1788201596.097038:0:412522:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d02100. 00000080:00000010:2.1:1788201596.097047:0:412522:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcc40. 00010000:00000010:2.1:1788201596.097053:0:412522:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a800. 00010000:00000010:2.1:1788201596.097056:0:412522:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:0.1:1788201596.097232:0:412513:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.097237:0:412513:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00010000:00000010:1.1:1788201596.123865:0:412523:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d200. 00010000:00000010:1.1:1788201596.123880:0:412523:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00000080:00000010:2.1:1788201596.126747:0:412522:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab47a940. 00000080:00000010:2.1:1788201596.126760:0:412522:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441bc0. 00000080:00000010:2.1:1788201596.126771:0:412522:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d02940. 00000080:00000010:2.1:1788201596.126775:0:412522:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc6c0. 00010000:00000010:2.1:1788201596.126782:0:412522:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.126787:0:412522:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:2.1:1788201596.129116:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab440000. 00000080:00000010:2.1:1788201596.129123:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcaa0. 00000080:00000010:2.1:1788201596.139120:0:412527:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab47e300. 00000080:00000010:0.1:1788201596.139121:0:412319:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab42b9c0. 00000080:00000010:2.1:1788201596.139131:0:412527:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc060. 00010000:00000010:2.1:1788201596.139140:0:412527:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858500. 00000080:00000010:0.1:1788201596.139142:0:412319:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dce0. 00010000:00000010:2.1:1788201596.139146:0:412527:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:2.1:1788201596.139150:0:412527:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a800. 00010000:00000010:2.1:1788201596.139153:0:412527:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00010000:00000010:2.1:1788201596.139155:0:412527:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.139157:0:412527:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00000080:00000010:2.1:1788201596.139163:0:412527:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab43a940. 00000080:00000010:2.1:1788201596.139166:0:412527:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcd40. 00000080:00000010:2.1:1788201596.139171:0:412527:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab479080. 00000080:00000010:2.1:1788201596.139175:0:412527:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcc40. 00000080:00000010:1.1:1788201596.140829:0:412523:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d31080. 00000080:00000010:1.1:1788201596.140839:0:412523:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbba0. 00000080:00000010:2.1:1788201596.145118:0:412524:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d00000. 00000080:00000010:2.1:1788201596.145127:0:412524:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc0e0. 00010000:00000010:0.1:1788201596.175304:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.175317:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00010000:00000010:2.1:1788201596.189917:0:412227:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.189930:0:412227:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00010000:00000010:2.1:1788201596.189935:0:412227:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a800. 00010000:00000010:2.1:1788201596.189938:0:412227:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00000080:00000010:0.1:1788201596.201128:0:412531:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab43ca40. 00000080:00000010:0.1:1788201596.201142:0:412531:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d220. 00010000:00000010:0.1:1788201596.205225:0:412531:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.205233:0:412531:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00010000:00000010:2.1:1788201596.207164:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.207173:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:2.1:1788201596.221866:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab47c200. 00000080:00000010:2.1:1788201596.221876:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcbe0. 00000080:00000010:2.1:1788201596.248309:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab439080. 00000080:00000010:2.1:1788201596.248318:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc6c0. 00000080:00000010:2.1:1788201596.264138:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d00840. 00000080:00000010:2.1:1788201596.264157:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcaa0. 00000080:00000010:0.1:1788201596.267017:0:412319:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab424200. 00000080:00000010:0.1:1788201596.267033:0:412319:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d740. 00010000:00000010:2.1:1788201596.305112:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.305117:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00010000:00000010:2.1:1788201596.305248:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a800. 00010000:00000010:2.1:1788201596.305252:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00010000:00000010:2.1:1788201596.312941:0:412227:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.312959:0:412227:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00000080:00000010:2.1:1788201596.312968:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d05280. 00000080:00000010:2.1:1788201596.312993:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fce20. 00010000:00000010:0.1:1788201596.316077:0:412536:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.316084:0:412536:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00000080:00000010:2.1:1788201596.319377:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab44eb40. 00000080:00000010:2.1:1788201596.319389:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc8c0. 00000080:00000010:2.1:1788201596.321334:0:256127:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab476b40. 00000080:00000010:2.1:1788201596.321342:0:256127:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc060. 00000080:00000010:0.1:1788201596.321528:0:412536:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab43d280. 00000080:00000010:0.1:1788201596.328516:0:412535:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc60. 00010000:00000010:2.1:1788201596.329116:0:412538:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.329123:0:412538:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:2.1:1788201596.363139:0:412543:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a800. 00010000:00000010:2.1:1788201596.363147:0:412543:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00010000:00000010:2.1:1788201596.363154:0:412543:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.363158:0:412543:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00010000:00000010:0.1:1788201596.368362:0:412541:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.368377:0:412541:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00000080:00000010:2.1:1788201596.370390:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab470840. 00000080:00000010:2.1:1788201596.370403:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc6c0. 00000080:00000010:2.1:1788201596.382126:0:412536:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab456b40. 00000080:00000010:2.1:1788201596.382144:0:412536:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcc40. 00000080:00000010:2.1:1788201596.382153:0:412536:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab475280. 00000080:00000010:2.1:1788201596.382156:0:412536:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcbe0. 00000080:00000010:0.1:1788201596.414813:0:12:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab43b180. 00000080:00000010:0.1:1788201596.414853:0:12:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d220. 00010000:00000010:2.1:1788201596.415295:0:412536:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.415305:0:412536:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:2.1:1788201596.431904:0:412549:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a800. 00010000:00000010:2.1:1788201596.431919:0:412549:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00010000:00000010:2.1:1788201596.447695:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.447704:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:2.1:1788201596.447711:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201596.447718:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646400. 00010000:00000010:0.1:1788201596.453215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.453229:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00000080:00000010:2.1:1788201596.465352:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab472940. 00000080:00000010:2.1:1788201596.465362:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc680. 00000080:00000010:2.1:1788201596.465369:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab44c200. 00000080:00000010:2.1:1788201596.465373:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc9c0. 00000080:00000010:0.1:1788201596.468005:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab448840. 00000080:00000010:0.1:1788201596.468029:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d240. 00000080:00000010:2.1:1788201596.474293:0:412555:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d11080. 00000080:00000010:2.1:1788201596.474304:0:412555:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc580. 00000080:00000010:2.1:1788201596.474313:0:412555:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d0a940. 00000080:00000010:2.1:1788201596.474318:0:412555:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441b60. 00010000:00000010:2.1:1788201596.474328:0:412555:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a800. 00010000:00000010:2.1:1788201596.474333:0:412555:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:0.1:1788201596.479464:0:412560:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.479477:0:412560:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00010000:00000010:2.1:1788201596.481964:0:412558:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a700. 00010000:00000010:2.1:1788201596.481979:0:412558:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00010000:00000010:2.1:1788201596.481983:0:412558:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.481986:0:412558:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00010000:00000010:2.1:1788201596.481991:0:412558:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.481993:0:412558:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201596.482000:0:412558:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d18000. 00000080:00000010:2.1:1788201596.482008:0:412558:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc700. 00000080:00000010:2.1:1788201596.495273:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab447380. 00000080:00000010:2.1:1788201596.495296:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc340. 00000080:00000010:2.1:1788201596.495305:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d2b180. 00000080:00000010:2.1:1788201596.495308:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fca40. 00000080:00000010:0.1:1788201596.504898:0:412559:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab422100. 00000080:00000010:0.1:1788201596.504926:0:412559:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dce0. 00000080:00000010:2.1:1788201596.508119:0:412231:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d139c0. 00000080:00000010:2.1:1788201596.508130:0:412231:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc400. 00010000:00000010:2.1:1788201596.508138:0:412231:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.508143:0:412231:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:2.1:1788201596.508149:0:412231:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.508152:0:412231:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00010000:00000010:2.1:1788201596.518156:0:412565:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.518168:0:412565:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:2.1:1788201596.542227:0:412549:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.542238:0:412549:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201596.542247:0:412549:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d21080. 00000080:00000010:2.1:1788201596.542254:0:412549:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc9c0. 00010000:00000010:0.1:1788201596.580749:0:412566:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.580761:0:412566:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:2.1:1788201596.593325:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d29080. 00000080:00000010:2.1:1788201596.593335:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc580. 00000080:00000010:2.1:1788201596.593586:0:412227:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d08000. 00000080:00000010:2.1:1788201596.593593:0:412227:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc700. 00010000:00000010:2.1:1788201596.593600:0:412227:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.593606:0:412227:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00000080:00000010:0.1:1788201596.605687:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab460840. 00000080:00000010:0.1:1788201596.605701:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc680. 00000080:00000010:0.1:1788201596.605713:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab456300. 00000080:00000010:0.1:1788201596.605721:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dfa0. 00010000:00000010:0.1:1788201596.611127:0:412567:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:2.1:1788201596.611130:0:412264:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:0.1:1788201596.611135:0:412567:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00010000:00000010:2.1:1788201596.611137:0:412264:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e600. 00000080:00000010:2.1:1788201596.611147:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d18840. 00000080:00000010:2.1:1788201596.611154:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcc40. 00000080:00000010:2.1:1788201596.627136:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d239c0. 00000080:00000010:2.1:1788201596.627143:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc9c0. 00010000:00000010:0.1:1788201596.628897:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201596.628904:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34e00. 00000080:00000010:0.1:1788201596.628913:0:412312:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab467380. 00000080:00000010:0.1:1788201596.628920:0:412312:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d260. 00010000:00000010:2.1:1788201596.636125:0:412570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.636135:0:412570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:2.1:1788201596.636226:0:412570:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.636228:0:412570:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00000080:00000010:0.1:1788201596.636565:0:412567:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab451080. 00000080:00000010:0.1:1788201596.636575:0:412567:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3df60. 00000080:00000010:2.1:1788201596.643119:0:412571:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d1a940. 00000080:00000010:2.1:1788201596.643131:0:412571:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc580. 00010000:00000010:0.1:1788201596.643200:0:412577:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201596.643207:0:412577:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00010000:00000010:0.1:1788201596.651911:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201596.651921:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00000080:00000010:2.1:1788201596.660594:0:26:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d1f380. 00000080:00000010:2.1:1788201596.660609:0:26:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc6c0. 00010000:00000010:2.1:1788201596.660618:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.660624:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:0.1:1788201596.710006:0:412575:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4618c0. 00000080:00000010:0.1:1788201596.710015:0:412575:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d8a0. 00000080:00000010:2.1:1788201596.712113:0:412579:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d1e300. 00000080:00000010:2.1:1788201596.712127:0:412579:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc9c0. 00010000:00000010:2.1:1788201596.712135:0:412579:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:2.1:1788201596.712140:0:412579:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00000080:00000010:1.1:1788201596.714260:0:412319:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab466b40. 00000080:00000010:1.1:1788201596.714273:0:412319:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d6e0. 00010000:00000010:2.1:1788201596.717128:0:412579:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.717139:0:412579:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201596.717148:0:412579:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d26b40. 00000080:00000010:2.1:1788201596.717155:0:412579:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc600. 00010000:00000010:1.1:1788201596.727578:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91ca00. 00010000:00000010:1.1:1788201596.727591:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00010000:00000010:1.1:1788201596.727596:0:412312:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201596.727599:0:412312:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:2.1:1788201596.733157:0:412582:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.733165:0:412582:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:1.1:1788201596.752202:0:412575:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab4698c0. 00000080:00000010:1.1:1788201596.752211:0:412575:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3db80. 00000080:00000010:2.1:1788201596.761229:0:412582:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d13180. 00000080:00000010:2.1:1788201596.761244:0:412582:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fce60. 00000080:00000010:1.1:1788201596.764080:0:412575:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab463180. 00000080:00000010:1.1:1788201596.764564:0:412575:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db760. 00010000:00000010:1.1:1788201596.773207:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201596.773219:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00000080:00000010:1.1:1788201596.784520:0:412583:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab46c200. 00000080:00000010:1.1:1788201596.784531:0:412583:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbba0. 00000080:00000010:2.1:1788201596.800117:0:412264:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d3c200. 00000080:00000010:2.1:1788201596.800127:0:412264:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441700. 00010000:00000010:1.1:1788201596.820197:0:412592:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201596.820211:0:412592:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00010000:00000010:2.1:1788201596.824457:0:412596:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.824474:0:412596:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e800. 00000080:00000010:2.1:1788201596.828152:0:412595:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d3b9c0. 00000080:00000010:2.1:1788201596.828167:0:412595:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc460. 00000080:00000010:1.1:1788201596.843270:0:412600:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab46a100. 00000080:00000010:1.1:1788201596.843293:0:412600:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db760. 00010000:00000010:2.1:1788201596.849115:0:412595:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.849128:0:412595:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201596.882246:0:412600:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d2b9c0. 00000080:00000010:2.1:1788201596.882263:0:412600:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc820. 00010000:00000010:1.1:1788201596.889116:0:412599:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201596.889125:0:412599:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00000080:00000010:1.1:1788201596.910118:0:412605:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab45b9c0. 00000080:00000010:1.1:1788201596.910130:0:412605:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db160. 00010000:00000010:2.1:1788201596.912220:0:412607:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.912231:0:412607:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:1.1:1788201596.930455:0:412587:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201596.930463:0:412587:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:2.1:1788201596.944135:0:412585:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d30840. 00000080:00000010:2.1:1788201596.944151:0:412585:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc960. 00000080:00000010:0.1:1788201596.944178:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab458000. 00000080:00000010:0.1:1788201596.944186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbdc0. 00010000:00000010:2.1:1788201596.999565:0:411072:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a200. 00010000:00000010:2.1:1788201596.999574:0:411072:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e400. 00010000:00000010:0.1:1788201597.007893:0:412616:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201597.007909:0:412616:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:2.1:1788201597.014470:0:412604:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d35280. 00000080:00000010:2.1:1788201597.014495:0:412604:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcc20. 00000080:00000010:0.1:1788201597.032053:0:412319:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab45eb40. 00000080:00000010:0.1:1788201597.032066:0:412319:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d100. 00010000:00000010:0.1:1788201597.068583:0:412637:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201597.068591:0:412637:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201597.093121:0:412630:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab458840. 00000080:00000010:0.1:1788201597.093133:0:412630:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d7a0. 00010000:00000010:0.1:1788201597.101480:0:412647:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201597.101493:0:412647:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34e00. 00000080:00000010:0.1:1788201597.101501:0:412647:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab45a940. 00000080:00000010:0.1:1788201597.101518:0:412647:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d040. 00010000:00000010:0.1:1788201597.144190:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201597.144204:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:0.1:1788201597.175666:0:412587:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d0e300. 00000080:00000010:0.1:1788201597.175674:0:412587:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d940. 00010000:00000010:1.1:1788201597.200139:0:412653:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201597.200147:0:412653:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34e00. 00000080:00000010:1.1:1788201597.222779:0:412665:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aab47eb40. 00000080:00000010:1.1:1788201597.222825:0:412665:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd00. 00010000:00000010:1.1:1788201597.279201:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201597.279215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:0.1:1788201597.298583:0:412672:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d14200. 00000080:00000010:0.1:1788201597.298594:0:412672:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db920. 00010000:00000010:0.1:1788201597.298612:0:412672:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00010000:00000010:0.1:1788201597.298618:0:412672:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:1.1:1788201597.312270:0:412675:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d14a40. 00000080:00000010:1.1:1788201597.312288:0:412675:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d940. 00010000:00000010:1.1:1788201597.351108:0:412319:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201597.351116:0:412319:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201597.370666:0:412691:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d12100. 00000080:00000010:1.1:1788201597.370678:0:412691:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db160. 00010000:00000010:1.1:1788201597.384164:0:412695:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201597.384179:0:412695:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201597.409072:0:412695:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d218c0. 00000080:00000010:1.1:1788201597.409088:0:412695:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbbc0. 00010000:00000010:2.1:1788201610.360568:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00000080:00000010:2.1:1788201610.365163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcd00. 00010000:00000010:2.1:1788201610.365181:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a000. 00010000:00000010:2.1:1788201610.369484:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f400. 00010000:00000010:1.1:1788201611.897295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00000080:00000010:0.1:1788201611.910684:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db400. 00000080:00000010:0.1:1788201611.910706:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d660. 00010000:00000010:0.1:1788201611.910714:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:1.1:1788201622.669118:0:414193:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00000080:00000010:1.1:1788201622.702342:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd80. 00010000:00000010:1.1:1788201622.702366:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737df00. 00010000:00000010:1.1:1788201622.702372:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00000080:00000010:1.1:1788201622.769483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fca40. 00010000:00000010:2.1:1788201622.962801:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f000. 00010000:00000010:0.1:1788201622.967304:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ad00. 00010000:00000010:0.1:1788201622.967318:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fe00. 00000080:00000010:0.1:1788201623.007181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc400. 00010000:00000010:0.1:1788201623.007204:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c800. 00010000:00000010:0.1:1788201623.007209:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00000080:00000010:0.1:1788201623.061219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd00. 00010000:00000010:1.1:1788201623.116135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00010000:00000010:0.1:1788201623.165432:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c100. 00010000:00000010:0.1:1788201623.165441:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:1.1:1788201623.179441:0:414199:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00000080:00000010:1.1:1788201623.195351:0:414200:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbc60. 00010000:00000010:1.1:1788201623.250775:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00010000:00000010:0.1:1788201623.310144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:1.1:1788201623.338136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35e00. 00000080:00000010:2.1:1788201623.362328:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db560. 00010000:00000010:1.1:1788201623.396278:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201623.396295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00010000:00000010:2.1:1788201623.427831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00000080:00000010:1.1:1788201623.458228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db000. 00010000:00000010:0.1:1788201623.519071:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:1.1:1788201623.538160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00010000:00000010:0.1:1788201623.546134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:1.1:1788201623.566555:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646000. 00000080:00000010:0.1:1788201623.588138:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db080. 00010000:00000010:1.1:1788201623.608654:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00000080:00000010:1.1:1788201623.612619:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d040. 00010000:00000010:1.1:1788201623.617429:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00010000:00000010:0.1:1788201623.702134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:1.1:1788201623.710129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00010000:00000010:0.1:1788201623.717361:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00010000:00000010:1.1:1788201623.744298:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00010000:00000010:1.1:1788201623.749137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00010000:00000010:1.1:1788201623.749150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fc00. 00010000:00000010:1.1:1788201623.749154:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c100. 00010000:00000010:1.1:1788201623.749158:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00010000:00000010:1.1:1788201623.749161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:1.1:1788201623.749163:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646e00. 00010000:00000010:1.1:1788201623.749166:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:1.1:1788201623.749167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00010000:00000010:0.1:1788201623.776155:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201623.776164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0e00. 00010000:00000010:1.1:1788201623.779316:0:414207:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d900. 00010000:00000010:1.1:1788201623.779321:0:414207:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00000080:00000010:0.1:1788201623.829251:0:414208:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db000. 00000080:00000010:0.1:1788201623.846825:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db9a0. 00010000:00000010:0.1:1788201623.857133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00000080:00000010:0.1:1788201623.857144:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d400. 00010000:00000010:2.1:1788201623.873162:0:414043:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f400. 00000080:00000010:2.1:1788201623.885135:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc1a0. 00010000:00000010:1.1:1788201624.019264:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c400. 00010000:00000010:1.1:1788201624.019280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34200. 00010000:00000010:1.1:1788201624.019286:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a500. 00010000:00000010:1.1:1788201624.019290:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f800. 00000080:00000010:2.1:1788201624.038405:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54b160. 00000080:00000010:2.1:1788201624.046420:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cde0. 00000080:00000010:2.1:1788201624.050284:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab5201640. 00000080:00000010:2.1:1788201624.061580:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab539c5c0. 00000080:00000010:2.1:1788201624.069245:0:395636:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab539c360. 00000080:00000010:2.1:1788201624.076225:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db200. 00000080:00000010:2.1:1788201624.082474:0:395636:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3c200. 00000080:00000010:2.1:1788201624.088232:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bbc54bfc0. 00000080:00000010:2.1:1788201624.093272:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7ab539c3c0. 00000080:00000010:2.1:1788201624.100852:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cfc0. 00000080:00000010:2.1:1788201624.148224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aa6e3cd40. 00000080:00000010:2.1:1788201624.148245:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d940. 00000080:00000010:2.1:1788201624.148262:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb80. 00010000:00000010:2.1:1788201624.210173:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d600. 00010000:00000010:2.1:1788201624.210186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00010000:00000010:2.1:1788201624.210192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647000. 00010000:00000010:2.1:1788201624.210196:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9800. 00010000:00000010:2.1:1788201624.210200:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee800. 00010000:00000010:2.1:1788201624.210204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24200. 00010000:00000010:2.1:1788201624.210210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030400. 00010000:00000010:2.1:1788201624.210215:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577600. 00010000:00000010:2.1:1788201624.210220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a900. 00010000:00000010:2.1:1788201624.210224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fc00. 00010000:00000010:2.1:1788201624.210228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5800. 00010000:00000010:2.1:1788201624.210232:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4c00. 00010000:00000010:2.1:1788201624.210235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9ce00. 00010000:00000010:2.1:1788201624.210238:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5600. 00010000:00000010:2.1:1788201624.210240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45fe00. 00010000:00000010:2.1:1788201624.210254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f800. 00010000:00000010:2.1:1788201624.210257:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031c00. 00010000:00000010:2.1:1788201624.210262:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030000. 00010000:00000010:2.1:1788201624.210266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef200. 00010000:00000010:2.1:1788201624.210268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee400. 00010000:00000010:2.1:1788201624.210272:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab989a00. 00010000:00000010:2.1:1788201624.210275:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4e00. 00010000:00000010:2.1:1788201624.210277:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80031a00. 00010000:00000010:2.1:1788201624.210280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030e00. 00010000:00000010:2.1:1788201624.210283:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e000. 00010000:00000010:2.1:1788201624.210286:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d500. 00010000:00000010:2.1:1788201624.210289:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b80030c00. 00010000:00000010:2.1:1788201624.210292:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00010000:00000010:2.1:1788201624.210296:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00010000:00000010:2.1:1788201624.210298:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788201624.210302:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ec00. 00010000:00000010:2.1:1788201624.210307:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:2.1:1788201624.210310:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1e00. 00010000:00000010:2.1:1788201624.210313:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c900. 00010000:00000010:2.1:1788201624.210315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1200. 00010000:00000010:2.1:1788201624.216906:0:414216:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cd00. 00010000:00000010:2.1:1788201624.216927:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00010000:00000010:2.1:1788201624.216931:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00010000:00000010:2.1:1788201624.216935:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0800. 00010000:00000010:2.1:1788201624.216938:0:414216:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c200. 00010000:00000010:2.1:1788201624.216941:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1000. 00010000:00000010:2.1:1788201624.216945:0:414216:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737dd00. 00010000:00000010:2.1:1788201624.216949:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35400. 00010000:00000010:2.1:1788201624.216954:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6000. 00010000:00000010:2.1:1788201624.216957:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697400. 00010000:00000010:2.1:1788201624.216965:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870600. 00010000:00000010:2.1:1788201624.216968:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870a00. 00010000:00000010:2.1:1788201624.216973:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac622400. 00010000:00000010:2.1:1788201624.216978:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7a00. 00010000:00000010:2.1:1788201624.216983:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac964a00. 00010000:00000010:2.1:1788201624.216986:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac964200. 00010000:00000010:2.1:1788201624.216989:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0600. 00010000:00000010:2.1:1788201624.216992:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0c00. 00010000:00000010:2.1:1788201624.216996:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9740000. 00010000:00000010:2.1:1788201624.217000:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577c00. 00010000:00000010:2.1:1788201624.217005:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82600. 00010000:00000010:2.1:1788201624.217012:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4600. 00010000:00000010:2.1:1788201624.217015:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4000. 00010000:00000010:2.1:1788201624.217018:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5c00. 00010000:00000010:2.1:1788201624.217021:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870e00. 00010000:00000010:2.1:1788201624.217024:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83c00. 00010000:00000010:2.1:1788201624.217029:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9741e00. 00010000:00000010:2.1:1788201624.217032:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82200. 00010000:00000010:2.1:1788201624.217036:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab989200. 00010000:00000010:2.1:1788201624.217041:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f400. 00010000:00000010:2.1:1788201624.217045:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9740c00. 00010000:00000010:2.1:1788201624.217050:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1800. 00010000:00000010:2.1:1788201624.217053:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697e00. 00010000:00000010:2.1:1788201624.217056:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0400. 00010000:00000010:2.1:1788201624.217059:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577200. 00010000:00000010:2.1:1788201624.217254:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14400. 00010000:00000010:2.1:1788201624.217261:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183a00. 00010000:00000010:2.1:1788201624.217266:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2c00. 00010000:00000010:2.1:1788201624.217271:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3c00. 00010000:00000010:2.1:1788201624.217273:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3800. 00010000:00000010:2.1:1788201624.217277:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2200. 00010000:00000010:2.1:1788201624.217280:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576e00. 00010000:00000010:2.1:1788201624.217284:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e200. 00010000:00000010:2.1:1788201624.217287:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3000. 00010000:00000010:2.1:1788201624.217291:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182600. 00010000:00000010:2.1:1788201624.217295:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183800. 00010000:00000010:2.1:1788201624.217297:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182e00. 00010000:00000010:2.1:1788201624.217300:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182000. 00010000:00000010:2.1:1788201624.217302:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15400. 00010000:00000010:2.1:1788201624.217305:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8400. 00010000:00000010:2.1:1788201624.217310:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8c00. 00010000:00000010:2.1:1788201624.217313:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577400. 00010000:00000010:2.1:1788201624.217316:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c14600. 00010000:00000010:2.1:1788201624.217319:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:2.1:1788201624.217322:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871400. 00010000:00000010:2.1:1788201624.217325:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1c00. 00010000:00000010:2.1:1788201624.217327:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1000. 00010000:00000010:2.1:1788201624.217330:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3200. 00010000:00000010:2.1:1788201624.217332:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00010000:00000010:2.1:1788201624.217335:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1a00. 00010000:00000010:2.1:1788201624.217339:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:2.1:1788201624.217342:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647200. 00010000:00000010:2.1:1788201624.217345:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647a00. 00010000:00000010:2.1:1788201624.217348:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13400. 00010000:00000010:2.1:1788201624.217353:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19aa00. 00010000:00000010:2.1:1788201624.217356:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af7000. 00010000:00000010:2.1:1788201624.217369:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045000. 00010000:00000010:2.1:1788201624.217374:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182200. 00010000:00000010:2.1:1788201624.217376:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2400. 00010000:00000010:2.1:1788201624.217379:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3400. 00010000:00000010:2.1:1788201624.217383:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2800. 00010000:00000010:2.1:1788201624.217386:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045600. 00010000:00000010:2.1:1788201624.217389:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045e00. 00010000:00000010:2.1:1788201624.217392:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646c00. 00010000:00000010:2.1:1788201624.217396:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac965200. 00010000:00000010:2.1:1788201624.217400:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149c200. 00010000:00000010:2.1:1788201624.217404:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac622e00. 00010000:00000010:2.1:1788201624.217408:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac964c00. 00010000:00000010:2.1:1788201624.217411:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac965800. 00010000:00000010:2.1:1788201624.217414:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13a00. 00010000:00000010:2.1:1788201624.217419:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f12200. 00010000:00000010:2.1:1788201624.217422:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f13600. 00010000:00000010:2.1:1788201624.217424:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183c00. 00010000:00000010:2.1:1788201624.217427:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183600. 00010000:00000010:2.1:1788201624.217431:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc1e00. 00010000:00000010:2.1:1788201624.217434:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2e00. 00010000:00000010:2.1:1788201624.217437:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f12600. 00010000:00000010:2.1:1788201624.217439:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ae00. 00010000:00000010:2.1:1788201624.217442:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577000. 00010000:00000010:2.1:1788201624.217445:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac964600. 00010000:00000010:2.1:1788201624.217449:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc0e00. 00010000:00000010:2.1:1788201624.217452:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc0400. 00010000:00000010:2.1:1788201624.217455:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f200. 00010000:00000010:2.1:1788201624.217460:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044e00. 00010000:00000010:2.1:1788201624.217464:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3a00. 00010000:00000010:2.1:1788201624.217467:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576c00. 00010000:00000010:2.1:1788201624.217470:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182400. 00010000:00000010:2.1:1788201624.217473:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6600. 00010000:00000010:2.1:1788201624.217477:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b200. 00010000:00000010:2.1:1788201624.217480:0:414216:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b800. 00010000:00000010:2.1:1788201624.217507:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a200. 00010000:00000010:2.1:1788201624.217512:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b000. 00010000:00000010:2.1:1788201624.217514:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183000. 00010000:00000010:2.1:1788201624.217518:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6200. 00010000:00000010:2.1:1788201624.217521:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b85f12e00. 00010000:00000010:2.1:1788201624.217537:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25200. 00010000:00000010:2.1:1788201624.217541:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187fa00. 00010000:00000010:2.1:1788201624.217545:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ee00. 00010000:00000010:2.1:1788201624.217547:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e200. 00010000:00000010:2.1:1788201624.217550:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6c00. 00010000:00000010:2.1:1788201624.217554:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183400. 00010000:00000010:2.1:1788201624.217557:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182c00. 00010000:00000010:2.1:1788201624.217561:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0e00. 00010000:00000010:2.1:1788201624.217563:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647c00. 00010000:00000010:2.1:1788201624.217566:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25600. 00010000:00000010:2.1:1788201624.217569:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b87697c00. 00010000:00000010:2.1:1788201624.217577:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1ee00. 00010000:00000010:2.1:1788201624.217581:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2182a00. 00010000:00000010:2.1:1788201624.217584:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb2183200. 00010000:00000010:2.1:1788201624.217592:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07800. 00010000:00000010:2.1:1788201624.217597:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9dc00. 00010000:00000010:2.1:1788201624.217602:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c800. 00010000:00000010:2.1:1788201624.217605:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2600. 00010000:00000010:2.1:1788201624.217607:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9ca00. 00010000:00000010:2.1:1788201624.217611:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef800. 00010000:00000010:2.1:1788201624.217614:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b86af6400. 00010000:00000010:2.1:1788201624.217617:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9741200. 00010000:00000010:2.1:1788201624.217620:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9741600. 00010000:00000010:2.1:1788201624.217623:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576a00. 00010000:00000010:2.1:1788201624.217625:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d3600. 00010000:00000010:2.1:1788201624.217630:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e800. 00010000:00000010:2.1:1788201624.217633:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6000. 00010000:00000010:2.1:1788201624.217636:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ba00. 00010000:00000010:2.1:1788201624.217639:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b600. 00010000:00000010:2.1:1788201624.217641:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:2.1:1788201624.217645:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eee00. 00010000:00000010:2.1:1788201624.217648:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1400. 00010000:00000010:2.1:1788201624.217651:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044800. 00010000:00000010:2.1:1788201624.217654:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eec00. 00010000:00000010:2.1:1788201624.217657:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:2.1:1788201624.217661:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac965a00. 00010000:00000010:2.1:1788201624.217664:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1fc00. 00010000:00000010:2.1:1788201624.217667:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8d2a00. 00010000:00000010:2.1:1788201624.217670:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac622000. 00010000:00000010:2.1:1788201624.217674:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac623c00. 00010000:00000010:2.1:1788201624.217677:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac622a00. 00010000:00000010:2.1:1788201624.217680:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac964400. 00010000:00000010:2.1:1788201624.217683:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac965c00. 00010000:00000010:2.1:1788201624.217685:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac964800. 00010000:00000010:2.1:1788201624.217688:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc1400. 00010000:00000010:2.1:1788201624.217692:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f600. 00010000:00000010:2.1:1788201624.217696:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a800. 00010000:00000010:2.1:1788201624.217699:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871c00. 00010000:00000010:2.1:1788201624.217701:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871000. 00010000:00000010:2.1:1788201624.217704:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc82e00. 00010000:00000010:2.1:1788201624.217708:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc83600. 00010000:00000010:2.1:1788201624.217715:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac623400. 00010000:00000010:2.1:1788201624.217719:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac623200. 00010000:00000010:2.1:1788201624.217722:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac622800. 00010000:00000010:2.1:1788201624.217725:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef600. 00010000:00000010:2.1:1788201624.217729:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee600. 00010000:00000010:2.1:1788201624.217732:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac964000. 00010000:00000010:2.1:1788201624.217735:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b8149c000. 00010000:00000010:2.1:1788201624.217738:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870800. 00010000:00000010:2.1:1788201624.217741:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6600. 00010000:00000010:2.1:1788201624.217744:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45e400. 00010000:00000010:2.1:1788201624.217748:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7800. 00010000:00000010:2.1:1788201624.217751:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6200. 00010000:00000010:2.1:1788201624.217754:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7400. 00010000:00000010:2.1:1788201624.217757:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab989000. 00010000:00000010:2.1:1788201624.217761:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab989e00. 00010000:00000010:2.1:1788201624.217763:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efa00. 00010000:00000010:2.1:1788201624.217766:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee000. 00010000:00000010:2.1:1788201624.217769:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef000. 00010000:00000010:2.1:1788201624.217772:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab988800. 00010000:00000010:2.1:1788201624.217776:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc1a00. 00010000:00000010:2.1:1788201624.217779:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a400. 00010000:00000010:2.1:1788201624.217782:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7000. 00010000:00000010:2.1:1788201624.217784:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871600. 00010000:00000010:2.1:1788201624.217787:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9740600. 00010000:00000010:2.1:1788201624.217790:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9740800. 00010000:00000010:2.1:1788201624.217793:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9740a00. 00010000:00000010:2.1:1788201624.217795:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab988000. 00010000:00000010:2.1:1788201624.217798:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab989800. 00010000:00000010:2.1:1788201624.217800:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab988e00. 00010000:00000010:2.1:1788201624.217803:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07600. 00010000:00000010:2.1:1788201624.217805:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07e00. 00010000:00000010:2.1:1788201624.217808:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc1200. 00010000:00000010:2.1:1788201624.217810:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc0000. 00010000:00000010:2.1:1788201624.217813:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa31d7500. 00010000:00000010:2.1:1788201624.217817:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc1800. 00010000:00000010:2.1:1788201624.217820:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a300. 00010000:00000010:2.1:1788201624.217822:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc0200. 00010000:00000010:2.1:1788201624.217824:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaacc0c00. 00010000:00000010:2.1:1788201624.217828:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06600. 00010000:00000010:2.1:1788201624.217831:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9741800. 00010000:00000010:2.1:1788201624.217834:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1ec00. 00010000:00000010:2.1:1788201624.217837:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e400. 00010000:00000010:2.1:1788201624.217840:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d000. 00010000:00000010:2.1:1788201624.217843:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d400. 00010000:00000010:2.1:1788201624.217847:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9741400. 00010000:00000010:2.1:1788201624.217850:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9741a00. 00010000:00000010:2.1:1788201624.217853:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870200. 00010000:00000010:2.1:1788201624.217856:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870c00. 00010000:00000010:2.1:1788201624.217858:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3871200. 00010000:00000010:2.1:1788201624.217860:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9740e00. 00010000:00000010:2.1:1788201624.217863:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb07000. 00010000:00000010:2.1:1788201624.217866:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e600. 00010000:00000010:2.1:1788201624.217869:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f200. 00010000:00000010:2.1:1788201624.217872:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab3870000. 00010000:00000010:2.1:1788201624.217882:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa3c15e00. 00010000:00000010:2.1:1788201624.217890:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1fe00. 00010000:00000010:2.1:1788201624.217892:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1fa00. 00010000:00000010:2.1:1788201624.217895:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25c00. 00010000:00000010:2.1:1788201624.217899:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24400. 00010000:00000010:2.1:1788201624.217903:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa9740200. 00010000:00000010:2.1:1788201624.217911:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d100. 00010000:00000010:2.1:1788201624.217914:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4a00. 00010000:00000010:2.1:1788201624.217917:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45ee00. 00010000:00000010:2.1:1788201624.217919:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aadb06000. 00010000:00000010:2.1:1788201624.217923:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5200. 00010000:00000010:2.1:1788201624.217925:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a4800. 00010000:00000010:2.1:1788201624.217928:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa6a5400. 00010000:00000010:2.1:1788201624.217934:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9600. 00010000:00000010:2.1:1788201624.217937:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25e00. 00010000:00000010:2.1:1788201624.217941:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f600. 00010000:00000010:2.1:1788201624.217944:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25400. 00010000:00000010:2.1:1788201624.217946:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa31d7200. 00010000:00000010:2.1:1788201624.217949:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b9f45f200. 00010000:00000010:2.1:1788201624.217952:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa31d7400. 00010000:00000010:2.1:1788201624.217955:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25a00. 00010000:00000010:2.1:1788201624.217958:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737dc00. 00010000:00000010:2.1:1788201624.217961:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24600. 00010000:00000010:2.1:1788201624.217964:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa31d7600. 00010000:00000010:2.1:1788201624.217967:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada25800. 00010000:00000010:2.1:1788201624.217970:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c300. 00010000:00000010:2.1:1788201624.217973:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aada24a00. 00010000:00000010:2.1:1788201624.217980:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab988200. 00010000:00000010:2.1:1788201624.217986:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa576600. 00010000:00000010:2.1:1788201624.217989:0:26:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d300. 00010000:00000010:2.1:1788201624.217993:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ee200. 00010000:00000010:1.1:1788201624.296132:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c500. 00010000:00000010:1.1:1788201624.296144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00010000:00000010:1.1:1788201624.296148:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:1.1:1788201624.296151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00010000:00000010:0.1:1788201624.308127:0:414220:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737da00. 00010000:00000010:0.1:1788201624.308144:0:414220:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:0.1:1788201624.308149:0:414220:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c100. 00010000:00000010:0.1:1788201624.308152:0:414220:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00000080:00000010:1.1:1788201626.078402:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db800. 00000080:00000010:2.1:1788201626.122302:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbe80. 00010000:00000010:2.1:1788201635.422192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ee00. 00000080:00000010:2.1:1788201635.509197:0:407529:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc2c0. 00000080:00000010:2.1:1788201635.509280:0:407529:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fca80. 00000080:00000010:2.1:1788201635.607141:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db3c0. 00000080:00000010:2.1:1788201635.621186:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc5c0. 00010000:00000010:0.1:1788201635.656571:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c300. 00010000:00000010:0.1:1788201635.656586:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00010000:00000010:0.1:1788201635.663826:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:1.1:1788201635.693213:0:414930:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db400. 00010000:00000010:0.1:1788201635.722214:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cd00. 00010000:00000010:0.1:1788201635.722230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0c00. 00010000:00000010:1.1:1788201635.730666:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0400. 00000080:00000010:2.1:1788201635.759853:0:256128:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db400. 00010000:00000010:1.1:1788201635.778219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d300. 00010000:00000010:1.1:1788201635.778226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00010000:00000010:0.1:1788201635.783304:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00000080:00000010:0.1:1788201635.802337:0:414932:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d9e0. 00010000:00000010:0.1:1788201635.850206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34a00. 00000080:00000010:1.1:1788201635.856247:0:414933:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d9e0. 00010000:00000010:1.1:1788201635.889279:0:414933:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00000080:00000010:1.1:1788201635.889293:0:414933:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db0e0. 00010000:00000010:1.1:1788201635.900605:0:414934:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cf00. 00010000:00000010:1.1:1788201635.900622:0:414934:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00010000:00000010:0.1:1788201635.932354:0:414934:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00000080:00000010:0.1:1788201635.932362:0:414934:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db760. 00010000:00000010:0.1:1788201635.936126:0:414935:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d100. 00010000:00000010:1.1:1788201635.988131:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cd00. 00010000:00000010:1.1:1788201635.988139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00000080:00000010:1.1:1788201635.991221:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d880. 00000080:00000010:1.1:1788201636.014296:0:414937:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc1a0. 00010000:00000010:1.1:1788201636.079182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0200. 00000080:00000010:1.1:1788201636.079191:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db560. 00010000:00000010:1.1:1788201636.132340:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0400. 00010000:00000010:1.1:1788201636.149134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35e00. 00010000:00000010:1.1:1788201636.183146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34c00. 00010000:00000010:1.1:1788201636.183171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c200. 00010000:00000010:1.1:1788201636.183176:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00010000:00000010:1.1:1788201636.183181:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35800. 00010000:00000010:1.1:1788201636.183183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00010000:00000010:1.1:1788201636.183188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737dc00. 00010000:00000010:1.1:1788201636.183191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34600. 00010000:00000010:2.1:1788201636.183845:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d100. 00010000:00000010:2.1:1788201636.183852:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00010000:00000010:2.1:1788201636.190020:0:256127:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c000. 00010000:00000010:2.1:1788201636.190033:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647600. 00000080:00000010:1.1:1788201636.192189:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d540. 00000080:00000010:1.1:1788201636.207219:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dac0. 00010000:00000010:0.1:1788201644.417820:0:415522:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647c00. 00000080:00000010:1.1:1788201644.480519:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de20. 00000080:00000010:0.1:1788201644.511196:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3ddc0. 00010000:00000010:0.1:1788201644.519211:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34800. 00000080:00000010:0.1:1788201644.623135:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de00. 00000080:00000010:0.1:1788201644.633140:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3ddc0. 00010000:00000010:0.1:1788201644.680149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35200. 00000080:00000010:1.1:1788201644.684172:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de00. 00000080:00000010:1.1:1788201644.699203:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db6c0. 00010000:00000010:1.1:1788201644.735161:0:415515:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201644.735169:0:415515:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1a00. 00010000:00000010:1.1:1788201644.749235:0:415515:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201644.771541:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91cc00. 00010000:00000010:1.1:1788201644.771565:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00010000:00000010:1.1:1788201644.788503:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d800. 00010000:00000010:1.1:1788201644.800156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1c00. 00000080:00000010:0.1:1788201644.823715:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db6a0. 00000080:00000010:0.1:1788201644.838181:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d5e0. 00010000:00000010:0.1:1788201644.873825:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e200. 00010000:00000010:1.1:1788201644.905155:0:415515:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0a00. 00000080:00000010:1.1:1788201644.911499:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db980. 00000080:00000010:1.1:1788201644.932667:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db560. 00010000:00000010:1.1:1788201644.941183:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00010000:00000010:1.1:1788201644.968149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1800. 00000080:00000010:1.1:1788201644.973830:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db980. 00000080:00000010:2.1:1788201644.994192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db980. 00010000:00000010:1.1:1788201645.038436:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0000. 00000080:00000010:1.1:1788201645.042478:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd80. 00000080:00000010:1.1:1788201645.065251:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd80. 00010000:00000010:1.1:1788201645.096306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c300. 00010000:00000010:1.1:1788201645.096315:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044e00. 00010000:00000010:0.1:1788201645.124494:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737dd00. 00010000:00000010:1.1:1788201645.147530:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34000. 00000080:00000010:1.1:1788201645.167211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf00. 00000080:00000010:1.1:1788201645.182164:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf00. 00010000:00000010:1.1:1788201645.226800:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c100. 00010000:00000010:1.1:1788201645.226814:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045600. 00010000:00000010:1.1:1788201645.268139:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c400. 00010000:00000010:1.1:1788201645.268154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045000. 00010000:00000010:0.1:1788201645.309137:0:415515:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737dc00. 00010000:00000010:0.1:1788201645.324325:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d400. 00000080:00000010:1.1:1788201645.354128:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d040. 00000080:00000010:1.1:1788201645.366419:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf00. 00010000:00000010:1.1:1788201645.409414:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3cd00. 00010000:00000010:1.1:1788201645.409423:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045800. 00010000:00000010:0.1:1788201645.459255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d500. 00010000:00000010:0.1:1788201645.473152:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9ca00. 00000080:00000010:1.1:1788201645.494826:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d1c0. 00000080:00000010:1.1:1788201645.517125:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb40. 00010000:00000010:1.1:1788201645.559456:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044c00. 00000080:00000010:1.1:1788201645.585236:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dd80. 00000080:00000010:1.1:1788201645.615415:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb40. 00010000:00000010:1.1:1788201645.701141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044600. 00000080:00000010:1.1:1788201645.744125:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db480. 00000080:00000010:1.1:1788201645.762192:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db660. 00010000:00000010:1.1:1788201645.776130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f200. 00000080:00000010:0.1:1788201645.795669:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d880. 00000080:00000010:0.1:1788201645.804878:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d7c0. 00010000:00000010:0.1:1788201645.833131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d200. 00000080:00000010:1.1:1788201645.861139:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db660. 00000080:00000010:1.1:1788201645.876421:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db480. 00010000:00000010:1.1:1788201645.894171:0:415515:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e400. 00010000:00000010:1.1:1788201645.913203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e400. 00000080:00000010:1.1:1788201645.913224:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db640. 00000080:00000010:0.1:1788201645.944033:0:13:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db640. 00010000:00000010:1.1:1788201645.957125:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4d00. 00010000:00000010:1.1:1788201645.957132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e800. 00010000:00000010:1.1:1788201645.974229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4d00. 00010000:00000010:1.1:1788201646.009132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1ee00. 00000080:00000010:1.1:1788201646.017274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd00. 00000080:00000010:1.1:1788201646.035227:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd00. 00010000:00000010:1.1:1788201646.068140:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f400. 00010000:00000010:1.1:1788201646.089132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f400. 00000080:00000010:0.1:1788201646.096138:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbfc0. 00000080:00000010:0.1:1788201646.105228:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d880. 00010000:00000010:0.1:1788201646.129163:0:415515:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c400. 00000080:00000010:0.1:1788201646.139272:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d420. 00000080:00000010:1.1:1788201646.167211:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db7a0. 00010000:00000010:1.1:1788201646.176144:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f800. 00010000:00000010:1.1:1788201646.198753:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f800. 00000080:00000010:1.1:1788201646.203711:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db800. 00000080:00000010:1.1:1788201646.223353:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db800. 00010000:00000010:1.1:1788201646.240129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9600. 00000080:00000010:1.1:1788201646.254139:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbee0. 00000080:00000010:1.1:1788201646.275242:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbee0. 00010000:00000010:1.1:1788201646.294252:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8400. 00000080:00000010:0.1:1788201646.333145:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbbc0. 00000080:00000010:1.1:1788201646.379299:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d420. 00010000:00000010:1.1:1788201646.410498:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b600. 00010000:00000010:1.1:1788201646.410526:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8000. 00010000:00000010:1.1:1788201646.459137:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b000. 00010000:00000010:1.1:1788201646.470324:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8800. 00010000:00000010:1.1:1788201646.498228:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9c00. 00000080:00000010:0.1:1788201646.516333:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbbc0. 00000080:00000010:1.1:1788201646.554768:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d960. 00010000:00000010:1.1:1788201646.580745:0:415515:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9c00. 00000080:00000010:1.1:1788201646.591352:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbde0. 00000080:00000010:2.1:1788201646.613154:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbde0. 00010000:00000010:0.1:1788201646.627249:0:415515:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84300. 00010000:00000010:0.1:1788201646.627257:0:415515:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9da00. 00010000:00000010:0.1:1788201646.659226:0:415515:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84b00. 00010000:00000010:1.1:1788201646.700295:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9cc00. 00010000:00000010:1.1:1788201646.724142:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8a00. 00000080:00000010:1.1:1788201646.732366:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db640. 00000080:00000010:1.1:1788201646.744151:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db980. 00010000:00000010:1.1:1788201646.760173:0:415515:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9400. 00000080:00000010:1.1:1788201646.766380:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db640. 00000080:00000010:1.1:1788201646.787294:0:415515:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db640. 00010000:00000010:0.1:1788201646.813442:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86e00. 00010000:00000010:0.1:1788201646.813462:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a800. 00010000:00000010:0.1:1788201646.828671:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ad00. 00010000:00000010:0.1:1788201646.859268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d800. 00000080:00000010:1.1:1788201646.877131:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb00. 00000080:00000010:1.1:1788201646.888233:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db6a0. 00010000:00000010:1.1:1788201646.905285:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b000. 00010000:00000010:1.1:1788201646.939120:0:415515:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a200. 00010000:00000010:1.1:1788201646.966126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ae00. 00000080:00000010:1.1:1788201646.966134:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb00. 00000080:00000010:1.1:1788201646.971343:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db220. 00010000:00000010:0.1:1788201647.042172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858300. 00010000:00000010:0.1:1788201647.042185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646200. 00000080:00000010:0.1:1788201647.052353:0:415524:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d940. 00010000:00000010:0.1:1788201647.076250:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9cc00. 00000080:00000010:0.1:1788201647.182139:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d940. 00000080:00000010:0.1:1788201647.205776:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3da40. 00000800:00000200:3.1:1788201654.090756:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.090782:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.090791:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.090796:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.091565:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.091576:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.091584:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.091592:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.091599:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092181:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092192:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092200:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092208:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092215:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092565:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092577:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092585:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092592:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.092599:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.093017:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.103809:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1445916 low water 1374194 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.421937:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487176 low water 1853564 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.421996:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487176 low water 1853564 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422008:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487176 low water 1853564 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422033:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487176 low water 1853564 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422042:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487176 low water 1853564 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422048:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487176 low water 1853564 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422091:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422775:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422791:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422802:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422812:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.422818:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.423562:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.423576:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.423584:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.423593:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.424214:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.424228:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.424238:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.424247:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 870672 low water 1661816 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.428525:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1509832 low water 1342236 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.889691:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 103680 low water 2045312 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.889980:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 103680 low water 2045312 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.890947:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 103680 low water 2045312 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.891661:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 103680 low water 2045312 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.892742:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 103680 low water 2045312 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.893040:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 103680 low water 2045312 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.893328:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 103680 low water 2045312 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894249:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 103680 low water 2045312 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894565:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 102400 low water 2045952 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894578:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 102400 low water 2045952 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894646:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 166316 low water 2013994 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894657:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 166316 low water 2013994 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894661:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 166316 low water 2013994 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894688:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 230232 low water 1982036 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894697:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 230232 low water 1982036 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894700:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 230232 low water 1982036 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894723:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 358064 low water 1918120 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894726:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 358064 low water 1918120 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894747:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 677644 low water 1758330 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894760:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 805476 low water 1694414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894774:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 869392 low water 1662456 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894787:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 933308 low water 1630498 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894805:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1061140 low water 1566582 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.894863:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1125056 low water 1534624 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.896896:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1180024 low water 1507140 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.896914:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1180024 low water 1507140 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.896924:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1180024 low water 1507140 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.896935:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1180024 low water 1507140 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.896962:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1180024 low water 1507140 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.896968:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1180024 low water 1507140 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.896994:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201654.898529:0:415527:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1445916 low water 1374194 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.857824:0:409463:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 395068 low water 1899618 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.857838:0:409463:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 395068 low water 1899618 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.857846:0:409463:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 395068 low water 1899618 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.857854:0:409463:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 395068 low water 1899618 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.857858:0:409463:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 395068 low water 1899618 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.858760:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.858773:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.858782:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.858801:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.858816:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.858822:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.859691:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.859707:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.859718:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.859729:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.859740:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.862782:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.862816:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.862829:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.862855:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.862865:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.863855:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551092 low water 1821606 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.865609:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.865840:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.865854:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.865864:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.865874:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.865880:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.867571:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.867586:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.867593:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.867601:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.867608:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382000 low water 1406152 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201656.869144:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1445916 low water 1374194 conn ffff9c7b847b0000 blocked idle queued 00010000:00000010:0.1:1788201658.191360:0:415531:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6561800. 00000080:00000010:0.1:1788201658.196231:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441400. 00010000:00000010:0.1:1788201658.200234:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1800. 00000080:00000010:1.1:1788201658.201149:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d4c0. 00010000:00000010:0.1:1788201658.234140:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:0.1:1788201658.471266:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cc00. 00010000:00000010:0.1:1788201658.471293:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00000800:00000200:3.1:1788201658.490437:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487592 low water 1853356 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.492565:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487592 low water 1853356 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.492624:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487592 low water 1853356 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.492638:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487592 low water 1853356 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.492651:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487592 low water 1853356 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.492662:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487592 low water 1853356 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.492677:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 487592 low water 1853356 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.493652:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.493669:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.493679:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.493688:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.493701:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.493707:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.493761:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.494493:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.494507:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.494514:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.494522:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.494529:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.495660:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.495678:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.495688:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.495699:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.495709:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497336:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497353:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497360:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497367:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497375:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497380:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497757:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497780:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497789:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497797:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.497804:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498237:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498253:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498264:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498274:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498284:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498722:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498737:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498748:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498758:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.498780:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.500052:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.500068:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.500077:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.500088:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.500097:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1318084 low water 1438110 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.508634:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 3299480 low water 447412 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.678591:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.678607:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.678615:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.678625:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.678633:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.679063:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.679076:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.679087:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.679096:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.679106:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.681734:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.681756:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.681778:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.681791:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.681800:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682291:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682316:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682327:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682336:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682346:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682756:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682784:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682796:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682807:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.682818:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.683818:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.683837:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.683849:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.683860:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.683871:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.683876:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.684286:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.684300:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.684321:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.684331:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.684342:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 871088 low water 1661608 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201658.685339:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1509832 low water 1342236 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.177668:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.177687:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.177697:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.178362:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.178375:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.178382:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.178666:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.178677:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.178685:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.179029:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.179040:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.179048:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.180742:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.180771:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.180781:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.181874:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.181888:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.181896:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.183114:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.183130:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.183138:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.183143:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.183719:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.184729:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.184769:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.184780:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.184787:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.185147:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.185159:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.185168:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.185780:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.185793:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.185802:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.187029:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.187045:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.187054:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.189608:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.189927:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.189938:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.189946:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.189952:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.190005:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1957244 low water 1118530 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.611949:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 231928 low water 1981188 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.612512:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 231928 low water 1981188 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.612931:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 231928 low water 1981188 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.612961:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 231928 low water 1981188 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.612975:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 231928 low water 1981188 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.612986:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 231928 low water 1981188 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.612996:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 231928 low water 1981188 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.613005:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 231928 low water 1981188 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.613053:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 295844 low water 1949230 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.613082:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.613984:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.613997:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.614006:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.614013:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 551508 low water 1821398 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.617959:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.617982:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.617995:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.618007:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.618012:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 998920 low water 1597692 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619311:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1062420 low water 1565942 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619326:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1062420 low water 1565942 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619334:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1062420 low water 1565942 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619341:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1062420 low water 1565942 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619350:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1062420 low water 1565942 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619354:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1062420 low water 1565942 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619399:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1126336 low water 1533984 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619417:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1190252 low water 1502026 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.619432:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.621487:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.621504:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.621513:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.621521:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.621529:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1254168 low water 1470068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201659.623471:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1509832 low water 1342236 conn ffff9c7b847b0000 blocked idle queued 00010000:00000010:3.1:1788201660.445403:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8be400. 00000080:00000010:3.1:1788201660.445411:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc400. 00010000:00000010:2.1:1788201660.466135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19bc00. 00010000:00000010:0.1:1788201660.637177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:2.1:1788201660.637232:0:25:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8bee00. 00010000:00000010:2.1:1788201660.637237:0:25:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8be600. 00010000:00000010:2.1:1788201660.637241:0:25:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8bfe00. 00010000:00000010:2.1:1788201660.637243:0:25:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8bfc00. 00010000:00000010:2.1:1788201660.637246:0:25:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8bec00. 00010000:00000010:2.1:1788201660.637248:0:25:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8be400. 00010000:00000010:2.1:1788201660.637250:0:25:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8bfa00. 00010000:00000010:0.1:1788201660.643366:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8bf200. 00010000:00000010:0.1:1788201660.643375:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8be200. 00010000:00000010:0.1:1788201660.643378:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1e00. 00010000:00000010:0.1:1788201660.643383:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0600. 00010000:00000010:0.1:1788201660.643385:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:0.1:1788201660.643388:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1400. 00010000:00000010:0.1:1788201660.643390:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0400. 00010000:00000010:0.1:1788201660.643393:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0200. 00010000:00000010:1.1:1788201660.723954:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0800. 00010000:00000010:1.1:1788201660.745605:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aab8bf400. 00010000:00000010:1.1:1788201660.750220:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863ce00. 00010000:00000010:1.1:1788201660.750235:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ac00. 00010000:00000010:1.1:1788201660.750239:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84c00. 00010000:00000010:1.1:1788201660.750245:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:3.1:1788201660.774427:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a800. 00010000:00000010:3.1:1788201660.774443:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa6560c00. 00010000:00000010:3.1:1788201660.774450:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a000. 00010000:00000010:1.1:1788201660.874398:0:415532:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac622c00. 00000080:00000010:1.1:1788201660.957131:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db240. 00000800:00000200:3.1:1788201667.003926:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 38728 low water 2077788 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.003947:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 38728 low water 2077788 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.003963:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 37448 low water 2078428 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.003968:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 37448 low water 2078428 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.004236:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 37448 low water 2078428 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.004253:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 36168 low water 2079068 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.004677:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 34888 low water 2079708 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.009418:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 71960 low water 2061172 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.009442:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 70680 low water 2061812 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.009447:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 70680 low water 2061812 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.009478:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 201072 low water 1996616 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.009488:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 201072 low water 1996616 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.009493:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 201072 low water 1996616 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.011629:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 220248 low water 1987028 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.013470:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 220248 low water 1987028 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.013853:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 220248 low water 1987028 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.014079:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 218968 low water 1987668 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.014108:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 218968 low water 1987668 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.017130:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 486140 low water 1854082 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.017152:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 486140 low water 1854082 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.017164:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 486140 low water 1854082 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.017169:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 486140 low water 1854082 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.023899:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 587128 low water 1803588 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.023925:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 587128 low water 1803588 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.023938:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 587128 low water 1803588 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.023944:0:415532:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 587128 low water 1803588 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.025078:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 587128 low water 1803588 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.025114:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 587128 low water 1803588 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.025128:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 587128 low water 1803588 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.031821:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 934832 low water 1629736 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.031845:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 934832 low water 1629736 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.031858:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 934832 low water 1629736 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.031868:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 934832 low water 1629736 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.031874:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 934832 low water 1629736 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.035803:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1126580 low water 1533862 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.035836:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1126580 low water 1533862 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.035847:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1126580 low water 1533862 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.035858:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1126580 low water 1533862 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.035864:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1126580 low water 1533862 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.035905:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1190496 low water 1501904 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.035944:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.036438:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.036451:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.036461:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.036470:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:2.1:1788201667.056806:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.064417:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.064432:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.064444:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.064454:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1382244 low water 1406030 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.064565:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1637908 low water 1278198 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.935751:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.935986:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -36524 low water 2115414 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.936286:0:32:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -37804 low water 2116054 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.942293:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -18628 low water 2106466 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.942303:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace -18628 low water 2106466 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.942469:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 548 low water 2096878 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.942474:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 548 low water 2096878 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.944449:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.944463:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.944468:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.944794:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.944808:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.945040:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.945051:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.945319:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.945331:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.945569:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.945580:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.945589:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 423676 low water 1885314 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.947671:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.947695:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.947706:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.947711:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.948283:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.948299:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.948309:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.948862:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.948877:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.948886:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.949487:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.949499:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.949754:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.949766:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.949774:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.950495:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.951581:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.951613:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 935004 low water 1629650 conn ffff9c7b847b0000 blocked idle queued 00000800:00000200:3.1:1788201667.952340:0:0:0:(socklnd_lib.c:489:ksocknal_write_space()) sk ffff9c7b832e0000 wspace 1509832 low water 1342236 conn ffff9c7b847b0000 blocked idle queued 00000080:00000010:1.1:1788201671.438369:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db520. 00010000:00000010:0.1:1788201671.475224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:1.1:1788201671.552133:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7c00. 00010000:00000010:1.1:1788201671.552147:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7800. 00010000:00000010:1.1:1788201671.552151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6800. 00010000:00000010:1.1:1788201671.552156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7e00. 00010000:00000010:1.1:1788201671.552161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6400. 00010000:00000010:1.1:1788201671.552166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201671.552172:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ac00. 00010000:00000010:2.1:1788201671.620495:0:415545:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7200. 00010000:00000010:2.1:1788201671.627134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ac00. 00010000:00000010:0.1:1788201671.793607:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:1.1:1788201671.818773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6600. 00010000:00000010:1.1:1788201672.032024:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9f600. 00010000:00000010:1.1:1788201672.201457:0:415549:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:1.1:1788201672.201468:0:415549:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84900. 00010000:00000010:1.1:1788201672.201474:0:415549:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00000080:00000010:1.1:1788201672.224041:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d8a0. 00000080:00000010:2.1:1788201673.128210:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d0c200. 00010000:00000010:2.1:1788201673.132136:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4ae00. 00010000:00000010:2.1:1788201673.132150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa577400. 00010000:00000010:0.1:1788201673.136132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0600. 00010000:00000010:0.1:1788201673.136150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1e00. 00010000:00000010:0.1:1788201673.136153:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1800. 00010000:00000010:0.1:1788201673.136156:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:0.1:1788201673.136159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1c00. 00010000:00000010:0.1:1788201673.136161:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0c00. 00010000:00000010:0.1:1788201673.136164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1a00. 00010000:00000010:0.1:1788201673.136166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00000080:00000010:0.1:1788201673.201167:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fcca0. 00000080:00000010:2.1:1788201673.993194:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d4b9c0. 00010000:00000010:2.1:1788201673.993213:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:2.1:1788201673.993219:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2ff00. 00010000:00000010:2.1:1788201673.993224:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0400. 00000080:00000010:3.1:1788201674.008456:0:415549:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db760. 00000080:00000010:3.1:1788201674.988194:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d28840. 00010000:00000010:2.1:1788201674.991334:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9e800. 00010000:00000010:2.1:1788201674.991343:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a000. 00010000:00000010:2.1:1788201674.991347:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac622e00. 00000080:00000010:2.1:1788201675.015301:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441820. 00000080:00000010:2.1:1788201675.020638:0:0:0:(super25.c:57:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff9c7aa5d0b180. 00010000:00000010:1.1:1788201677.018135:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a600. 00010000:00000010:1.1:1788201677.018151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aac623a00. 00010000:00000010:1.1:1788201677.018171:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cd00. 00010000:00000010:1.1:1788201677.018175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19aa00. 00010000:00000010:1.1:1788201677.022198:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84d00. 00010000:00000010:1.1:1788201677.022218:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1000. 00010000:00000010:0.1:1788201682.325773:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:1.1:1788201682.368132:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:0.1:1788201682.420239:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:0.1:1788201682.470734:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fd00. 00010000:00000010:0.1:1788201682.470751:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:1.1:1788201682.540133:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:1.1:1788201682.540150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201682.601428:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201682.678149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201682.724351:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:0.1:1788201682.771614:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:1.1:1788201682.792400:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:1.1:1788201682.792413:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201682.824300:0:416130:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201682.849254:0:416130:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:1.1:1788201682.849261:0:416130:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:0.1:1788201682.878280:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fd00. 00010000:00000010:0.1:1788201682.878288:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:0.1:1788201682.909787:0:416131:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:1.1:1788201682.925176:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:1.1:1788201682.925184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201682.944560:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201682.973831:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:0.1:1788201683.039263:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:0.1:1788201683.059135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:1.1:1788201683.092197:0:416130:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201683.113044:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201683.154357:0:416130:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:1.1:1788201683.154366:0:416130:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:0.1:1788201683.206136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:0.1:1788201683.225128:0:416130:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:1.1:1788201683.290192:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:0.1:1788201683.358146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7c00. 00010000:00000010:0.1:1788201683.409206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:0.1:1788201683.474270:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9ee00. 00010000:00000010:1.1:1788201683.508198:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201683.583664:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:1.1:1788201683.583673:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201683.621757:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201683.656718:0:1353:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:3.1:1788201683.673838:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858500. 00010000:00000010:3.1:1788201683.673846:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9fe00. 00010000:00000010:1.1:1788201683.726240:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6200. 00010000:00000010:1.1:1788201683.740139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:0.1:1788201683.773393:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fd00. 00010000:00000010:0.1:1788201683.773400:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:0.1:1788201683.801243:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1200. 00010000:00000010:1.1:1788201683.833120:0:416130:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:1.1:1788201683.844110:0:416130:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:3.1:1788201684.347683:0:32:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84a00. 00010000:00000010:3.1:1788201684.347691:0:32:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d800. 00000080:00000010:3.1:1788201684.365990:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa4412a0. 00010000:00000010:3.1:1788201684.444076:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858500. 00010000:00000010:3.1:1788201684.444106:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9fe00. 00000080:00000010:3.1:1788201684.480232:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa4412a0. 00010000:00000010:3.1:1788201684.656141:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9ec00. 00000080:00000010:3.1:1788201684.656158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441a20. 00010000:00000010:1.1:1788201684.672656:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6400. 00010000:00000010:2.1:1788201685.180135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9ea00. 00010000:00000010:0.1:1788201685.194211:0:256133:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9ec00. 00010000:00000010:3.1:1788201685.203817:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fd00. 00010000:00000010:3.1:1788201685.203828:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9fa00. 00010000:00000010:3.1:1788201685.203832:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863ca00. 00010000:00000010:3.1:1788201685.203836:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:1.1:1788201685.211139:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84100. 00010000:00000010:1.1:1788201685.211149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0200. 00000080:00000010:1.1:1788201685.252130:0:416170:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db7e0. 00000080:00000010:1.1:1788201685.260142:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441d80. 00010000:00000010:0.1:1788201685.329536:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19bc00. 00010000:00000010:1.1:1788201685.360143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201685.360160:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858500. 00010000:00000010:1.1:1788201685.360165:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9fe00. 00010000:00000010:1.1:1788201685.360168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ac00. 00010000:00000010:1.1:1788201685.363351:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c900. 00010000:00000010:1.1:1788201685.363368:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19aa00. 00010000:00000010:2.1:1788201685.383018:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c600. 00010000:00000010:2.1:1788201685.383026:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00000080:00000010:0.1:1788201685.518139:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db520. 00000080:00000010:0.1:1788201685.532194:0:416173:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db7e0. 00010000:00000010:0.1:1788201685.564887:0:416173:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1e00. 00000080:00000010:0.1:1788201685.564901:0:416173:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dc60. 00010000:00000010:0.1:1788201685.572219:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1800. 00010000:00000010:1.1:1788201691.107312:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19bc00. 00010000:00000010:1.1:1788201691.128117:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19bc00. 00010000:00000010:1.1:1788201691.148112:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19bc00. 00010000:00000010:1.1:1788201691.165125:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19bc00. 00010000:00000010:1.1:1788201691.187137:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:1.1:1788201691.187146:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19bc00. 00010000:00000010:1.1:1788201691.201123:0:416780:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:1.1:1788201691.215130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:1.1:1788201691.224113:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:1.1:1788201691.238113:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:1.1:1788201691.265139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:1.1:1788201691.274255:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:1.1:1788201691.274266:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.301136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.322134:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:1.1:1788201691.322143:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.342691:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:1.1:1788201691.342702:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.363129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.376109:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:1.1:1788201691.376118:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.418139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:0.1:1788201691.444122:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:0.1:1788201691.479126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:1.1:1788201691.495280:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.520244:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.542126:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.557123:0:416780:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:1.1:1788201691.557132:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.588134:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.604107:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.620125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.640129:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:1.1:1788201691.649123:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:1.1:1788201691.654139:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:3.1:1788201691.668140:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:3.1:1788201691.691112:0:416780:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858300. 00010000:00000010:3.1:1788201691.691118:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9f800. 00010000:00000010:3.1:1788201691.710501:0:256127:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9f800. 00010000:00000010:1.1:1788201691.741145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:1.1:1788201691.761586:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:1.1:1788201691.761594:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:0.1:1788201691.795348:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:0.1:1788201691.808130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:0.1:1788201691.827131:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fe00. 00010000:00000010:0.1:1788201691.827138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:0.1:1788201691.842112:0:416780:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0a00. 00010000:00000010:0.1:1788201691.863127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1400. 00010000:00000010:1.1:1788201692.002147:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:1.1:1788201692.002157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00000080:00000010:3.1:1788201692.009107:0:416790:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441500. 00010000:00000010:3.1:1788201692.035128:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858700. 00010000:00000010:3.1:1788201692.035140:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9f400. 00000080:00000010:0.1:1788201692.042124:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa4413e0. 00010000:00000010:0.1:1788201692.078118:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863cb00. 00010000:00000010:0.1:1788201692.078125:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d6600. 00010000:00000010:1.1:1788201692.087118:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9ec00. 00010000:00000010:1.1:1788201692.095131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9e600. 00010000:00000010:1.1:1788201692.100116:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9f400. 00010000:00000010:1.1:1788201692.100124:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858600. 00010000:00000010:1.1:1788201692.100131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9e000. 00010000:00000010:1.1:1788201692.100137:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858700. 00010000:00000010:1.1:1788201692.100150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:2.1:1788201692.104124:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858900. 00010000:00000010:2.1:1788201692.104130:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00000080:00000010:1.1:1788201692.127139:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc300. 00000080:00000010:1.1:1788201692.131156:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441980. 00010000:00000010:1.1:1788201692.148017:0:256134:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1400. 00010000:00000010:0.1:1788201692.158139:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c800. 00010000:00000010:0.1:1788201692.158155:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:0.1:1788201692.158159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1000. 00000080:00000010:0.1:1788201692.175148:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3ddc0. 00010000:00000010:1.1:1788201692.192307:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9ea00. 00010000:00000010:1.1:1788201692.201136:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19be00. 00010000:00000010:1.1:1788201692.201148:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef600. 00010000:00000010:1.1:1788201692.201151:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7ef400. 00010000:00000010:1.1:1788201692.201154:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eec00. 00010000:00000010:1.1:1788201692.201157:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7eee00. 00010000:00000010:1.1:1788201692.201159:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaa7efe00. 00010000:00000010:0.1:1788201708.402121:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f700. 00010000:00000010:0.1:1788201708.402127:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0200. 00000800:00000200:1.1:1788201750.620248:0:256124:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:1.1:1788201750.620252:0:256124:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:1.1:1788201750.620255:0:256124:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000800:00000200:1.1:1788201753.437364:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201753.437610:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201756.510343:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201756.510882:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201756.511404:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201756.511546:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201756.511647:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201761.628757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201761.629228:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201761.629575:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201761.629767:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201761.629909:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201766.748852:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201766.748860:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 4, 2 secs later) 00000800:00000200:1.1:1788201766.749753:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201766.749756:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 4, 2 secs later) 00000800:00000200:1.1:1788201766.750086:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201766.750109:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 4, 2 secs later) 00000800:00000200:1.1:1788201766.750356:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201766.750358:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 4, 2 secs later) 00000800:00000200:1.1:1788201766.750664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201766.750666:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 4, 2 secs later) 00000800:00000200:1.1:1788201771.868886:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201771.869280:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201771.869581:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201771.869758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201771.869947:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201776.989338:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201776.989342:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 6 secs later) 00000800:00000200:1.1:1788201776.989703:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201776.989723:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 6 secs later) 00000800:00000200:1.1:1788201776.990033:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201776.990037:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 6 secs later) 00000800:00000200:1.1:1788201776.990383:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201776.990386:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 6 secs later) 00000800:00000200:1.1:1788201776.990702:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201776.990722:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 6 secs later) 00000800:00000200:1.1:1788201782.108801:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201782.108808:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 1 secs later) 00000800:00000200:1.1:1788201782.109630:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201782.109632:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 1 secs later) 00000800:00000200:1.1:1788201782.109937:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201782.109939:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 1 secs later) 00000800:00000200:1.1:1788201782.110263:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201782.110266:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 1 secs later) 00000800:00000200:1.1:1788201782.110654:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201782.110656:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 8, 1 secs later) 00000800:00000200:1.1:1788201787.228793:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201787.229203:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201787.229574:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201787.229754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201787.229913:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201792.348790:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201792.348797:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 14 secs later) 00000800:00000200:1.1:1788201792.349656:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201792.349659:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 14 secs later) 00000800:00000200:1.1:1788201792.349995:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201792.349998:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 14 secs later) 00000800:00000200:1.1:1788201792.350271:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201792.350273:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 14 secs later) 00000800:00000200:1.1:1788201792.350592:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201792.350595:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 14 secs later) 00000800:00000200:1.1:1788201797.468781:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201797.468786:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 9 secs later) 00000800:00000200:1.1:1788201797.469477:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201797.469485:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 9 secs later) 00000800:00000200:1.1:1788201797.469788:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201797.469790:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 9 secs later) 00000800:00000200:1.1:1788201797.470019:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201797.470021:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 9 secs later) 00000800:00000200:1.1:1788201797.470218:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201797.470220:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 9 secs later) 00000800:00000200:1.1:1788201802.588719:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201802.588726:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 3 secs later) 00000800:00000200:1.1:1788201802.589401:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201802.589403:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 3 secs later) 00000800:00000200:1.1:1788201802.589722:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201802.589724:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 3 secs later) 00000800:00000200:1.1:1788201802.590038:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201802.590040:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 3 secs later) 00000800:00000200:1.1:1788201802.590375:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201802.590377:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 16, 3 secs later) 00000800:00000200:1.1:1788201807.708671:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201807.709019:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201807.709415:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201807.709594:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201807.709739:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201812.828778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201812.828784:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 30 secs later) 00000800:00000200:1.1:1788201812.829727:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201812.829729:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 30 secs later) 00000800:00000200:1.1:1788201812.829980:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201812.829982:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 30 secs later) 00000800:00000200:1.1:1788201812.830208:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201812.830209:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 30 secs later) 00000800:00000200:1.1:1788201812.830547:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201812.830548:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 30 secs later) 00000800:00000200:1.1:1788201817.948766:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201817.948772:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 25 secs later) 00000800:00000200:1.1:1788201817.949191:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201817.949193:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 25 secs later) 00000800:00000200:1.1:1788201817.949668:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201817.949670:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 25 secs later) 00000800:00000200:1.1:1788201817.950415:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201817.950417:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 25 secs later) 00000800:00000200:1.1:1788201817.951041:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201817.951043:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 25 secs later) 00000800:00000200:1.1:1788201823.068731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201823.068736:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 20 secs later) 00000800:00000200:1.1:1788201823.069486:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201823.069489:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 20 secs later) 00000800:00000200:1.1:1788201823.069773:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201823.069774:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 20 secs later) 00000800:00000200:1.1:1788201823.070027:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201823.070029:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 20 secs later) 00000800:00000200:1.1:1788201823.070283:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201823.070284:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 20 secs later) 00000800:00000200:1.1:1788201828.188928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201828.188936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 15 secs later) 00000800:00000200:1.1:1788201828.189502:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201828.189505:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 15 secs later) 00000800:00000200:1.1:1788201828.189841:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201828.189844:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 15 secs later) 00000800:00000200:1.1:1788201828.190203:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201828.190205:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 15 secs later) 00000800:00000200:1.1:1788201828.190544:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201828.190547:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 15 secs later) 00000800:00000200:3.1:1788201833.308779:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201833.308785:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 10 secs later) 00000800:00000200:3.1:1788201833.309479:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201833.309481:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 10 secs later) 00000800:00000200:3.1:1788201833.309750:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201833.309753:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 10 secs later) 00000800:00000200:3.1:1788201833.309972:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201833.309974:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 10 secs later) 00000800:00000200:3.1:1788201833.310248:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201833.310250:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 10 secs later) 00000800:00000200:1.1:1788201838.428720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201838.428725:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 5 secs later) 00000800:00000200:1.1:1788201838.429406:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201838.429408:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 5 secs later) 00000800:00000200:1.1:1788201838.429632:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201838.429634:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 5 secs later) 00000800:00000200:1.1:1788201838.429838:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201838.429840:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 5 secs later) 00000800:00000200:1.1:1788201838.430082:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201838.430084:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 32, 5 secs later) 00000800:00000200:1.1:1788201843.548676:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201843.549060:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201843.549499:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201843.549754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201843.550578:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201848.669306:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201848.669310:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201848.669737:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201848.669740:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201848.670057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201848.670060:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201848.670409:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201848.670411:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201848.670645:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201848.670647:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788201853.789124:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201853.789130:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201853.789905:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201853.789907:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201853.790110:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201853.790112:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201853.790296:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201853.790298:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201853.790545:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201853.790547:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201858.908842:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201858.908849:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788201858.909453:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201858.909454:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788201858.909691:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201858.909692:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788201858.909876:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201858.909878:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788201858.910052:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201858.910054:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201864.028760:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201864.028766:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201864.029393:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201864.029394:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201864.029576:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201864.029578:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201864.029752:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201864.029753:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201864.029925:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201864.029926:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201869.149013:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201869.149018:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788201869.150080:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201869.150083:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788201869.150442:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201869.150445:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788201869.150726:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201869.150729:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788201869.151224:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201869.151228:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788201874.268834:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201874.268842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788201874.269409:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201874.269411:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788201874.269605:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201874.269607:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788201874.269781:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201874.269782:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788201874.270103:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201874.270105:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788201879.388675:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201879.388681:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788201879.389593:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201879.389599:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788201879.389924:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201879.389941:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788201879.390221:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201879.390222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788201879.390413:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201879.390414:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788201884.508762:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201884.508765:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788201884.509455:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201884.509457:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788201884.509698:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201884.509700:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788201884.509903:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201884.509905:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788201884.510204:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201884.510206:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788201889.628676:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201889.628683:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788201889.629449:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201889.629453:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788201889.629802:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201889.629804:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788201889.630143:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201889.630145:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788201889.630326:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201889.630328:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788201894.749053:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201894.749067:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201894.749726:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201894.749728:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201894.749950:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201894.749952:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201894.750162:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201894.750164:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201894.750380:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201894.750382:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201899.868762:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201899.868770:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201899.869470:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201899.869474:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201899.869799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201899.869801:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201899.870131:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201899.870133:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201899.870434:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201899.870437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201904.989044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201904.989054:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788201904.990074:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201904.990077:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788201904.990336:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201904.990339:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788201904.990649:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201904.990651:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788201904.990839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201904.990840:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788201910.108764:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201910.109155:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201910.109340:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201910.109528:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201910.109660:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201915.229174:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201915.229179:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201915.230076:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201915.230079:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201915.230515:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201915.230517:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201915.230769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201915.230771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201915.230968:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201915.230970:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788201920.348958:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201920.348969:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201920.349922:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201920.349924:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201920.350181:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201920.350183:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201920.350422:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201920.350424:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788201920.350726:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201920.350728:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788201925.469117:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201925.469121:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201925.470292:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201925.470294:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201925.470551:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201925.470553:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201925.470812:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201925.470814:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201925.471034:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201925.471036:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201930.589257:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201930.589274:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788201930.590544:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201930.590546:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788201930.590915:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201930.590918:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788201930.591268:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201930.591271:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788201930.591639:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201930.591643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788201935.708742:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201935.708746:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788201935.709442:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201935.709445:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788201935.709768:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201935.709770:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788201935.710014:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201935.710016:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788201935.710225:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201935.710226:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788201940.828625:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201940.828629:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788201940.829130:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201940.829133:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788201940.829336:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201940.829338:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788201940.829519:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201940.829521:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788201940.829699:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201940.829701:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788201945.948664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201945.948667:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788201945.949287:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201945.949288:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788201945.949472:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201945.949474:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788201945.949649:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201945.949651:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788201945.949850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201945.949852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788201951.068754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201951.068759:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788201951.069484:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201951.069487:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788201951.069860:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201951.069862:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788201951.070137:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201951.070139:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788201951.070345:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201951.070346:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788201956.188889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201956.188895:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788201956.189631:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201956.189633:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788201956.189847:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201956.189848:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788201956.190088:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201956.190110:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788201956.190398:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788201956.190400:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788201961.308775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201961.308782:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201961.309489:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201961.309491:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201961.309788:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201961.309790:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201961.310087:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201961.310108:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201961.310461:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201961.310464:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788201966.428762:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201966.428769:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201966.429464:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201966.429465:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201966.429670:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201966.429673:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201966.429915:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201966.429917:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201966.430117:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201966.430119:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788201971.548767:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201971.548772:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788201971.549404:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201971.549406:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788201971.549597:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201971.549600:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788201971.549811:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201971.549813:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788201971.549987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201971.549988:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788201976.668842:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201976.669261:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201976.669422:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201976.669585:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201976.669754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201981.788827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201981.788835:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201981.789713:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201981.789716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201981.790033:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201981.790035:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201981.790330:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201981.790332:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201981.790623:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201981.790625:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788201986.908963:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201986.908971:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788201986.909805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201986.909809:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788201986.910173:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201986.910176:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788201986.910500:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201986.910503:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788201986.910819:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201986.910822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788201992.028801:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201992.028806:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201992.029451:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201992.029453:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201992.029645:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201992.029647:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201992.029822:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201992.029823:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201992.030050:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201992.030052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788201997.149234:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201997.149246:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201997.149814:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201997.149816:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201997.150039:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201997.150040:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201997.150257:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201997.150259:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788201997.150485:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788201997.150487:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202002.268930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202002.268941:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202002.269709:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202002.269712:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202002.270024:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202002.270026:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202002.270295:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202002.270298:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202002.270528:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202002.270531:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788202007.388872:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202007.388880:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202007.389441:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202007.389444:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202007.389761:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202007.389764:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202007.390057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202007.390060:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202007.390370:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202007.390372:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202012.509071:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202012.509111:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202012.509821:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202012.509823:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202012.510125:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202012.510127:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202012.510363:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202012.510365:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202012.510629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202012.510632:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202017.628758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202017.628764:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202017.629425:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202017.629430:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202017.629826:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202017.629828:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202017.630079:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202017.630081:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202017.630279:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202017.630280:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202022.748661:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202022.748669:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202022.749354:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202022.749357:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202022.749809:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202022.749812:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202022.750085:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202022.750087:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202022.750314:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202022.750316:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202027.868812:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202027.868821:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202027.869511:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202027.869514:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202027.869831:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202027.869835:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202027.870170:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202027.870172:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202027.870481:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202027.870483:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202032.988787:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202032.988791:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202032.989403:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202032.989405:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202032.989620:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202032.989622:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202032.989835:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202032.989837:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202032.990066:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202032.990068:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202038.108836:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202038.108841:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202038.109544:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202038.109547:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202038.109893:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202038.109896:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202038.110259:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202038.110262:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202038.110578:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202038.110581:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202043.228790:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202043.229239:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202043.229477:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202043.229633:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202043.229769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202048.349282:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202048.349285:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202048.349641:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202048.349644:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202048.349949:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202048.349951:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202048.350277:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202048.350278:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202048.350942:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202048.350945:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202053.468843:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202053.468850:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202053.469705:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202053.469710:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202053.470015:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202053.470017:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202053.470265:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202053.470267:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202053.470578:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202053.470580:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202058.588785:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202058.588790:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788202058.589528:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202058.589530:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788202058.589715:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202058.589717:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788202058.589942:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202058.589943:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788202058.590193:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202058.590195:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788202063.708601:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202063.708604:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202063.709033:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202063.709035:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202063.709257:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202063.709258:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202063.709436:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202063.709437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202063.709679:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202063.709681:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202068.828770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202068.828776:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202068.829399:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202068.829401:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202068.829593:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202068.829595:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202068.829826:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202068.829828:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202068.830126:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202068.830128:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202073.949133:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202073.949140:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202073.949928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202073.949987:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202073.950378:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202073.950381:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202073.950943:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202073.950946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202073.951776:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202073.951779:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202079.068774:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202079.068780:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202079.069386:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202079.069389:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202079.069683:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202079.069685:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202079.069943:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202079.069946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202079.070233:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202079.070235:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202084.188648:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202084.188664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202084.189193:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202084.189194:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202084.189356:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202084.189358:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202084.189552:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202084.189554:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202084.189804:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202084.189806:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202089.308645:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202089.308649:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202089.309150:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202089.309151:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202089.309362:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202089.309364:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202089.309545:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202089.309547:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202089.309710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202089.309713:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202094.428750:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202094.428757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202094.429611:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202094.429614:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202094.429837:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202094.429838:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202094.429996:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202094.429998:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202094.430219:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202094.430220:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202099.548713:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202099.548716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788202099.549235:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202099.549238:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788202099.549446:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202099.549447:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788202099.549605:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202099.549608:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788202099.549757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202099.549758:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788202104.668659:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202104.668664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202104.669230:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202104.669231:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202104.669491:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202104.669493:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202104.669699:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202104.669701:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202104.669850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202104.669851:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202109.788800:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202109.789299:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202109.789470:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202109.789646:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202109.789806:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202114.908827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202114.908850:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202114.909682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202114.909687:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202114.910057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202114.910060:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202114.910495:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202114.910499:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202114.910782:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202114.910785:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202120.029234:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202120.029247:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202120.029937:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202120.029940:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202120.030316:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202120.030319:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202120.030666:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202120.030669:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202120.031026:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202120.031030:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202125.148937:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202125.148946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202125.150012:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202125.150015:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202125.150371:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202125.150374:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202125.150695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202125.150698:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202125.151140:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202125.151143:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202130.269193:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202130.269200:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202130.269907:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202130.269910:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202130.270258:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202130.270260:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202130.270590:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202130.270592:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202130.270935:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202130.270937:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788202135.389164:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202135.389170:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788202135.390238:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202135.390241:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788202135.390540:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202135.390542:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788202135.390742:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202135.390744:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788202135.391075:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202135.391078:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788202140.508769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202140.508775:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202140.509441:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202140.509444:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202140.509727:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202140.509729:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202140.509916:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202140.509918:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202140.510164:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202140.510167:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202145.628907:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202145.628914:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202145.629636:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202145.629639:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202145.629959:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202145.629961:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202145.630292:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202145.630295:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202145.630557:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202145.630559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202150.748782:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202150.748787:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202150.749436:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202150.749442:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202150.749816:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202150.749818:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202150.750163:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202150.750166:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202150.750385:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202150.750387:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202155.868766:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202155.868773:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202155.869359:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202155.869363:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202155.869581:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202155.869583:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202155.869763:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202155.869765:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202155.869936:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202155.869938:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202160.988884:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202160.988891:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202160.989536:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202160.989539:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202160.989836:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202160.989839:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202160.990162:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202160.990164:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202160.990463:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202160.990466:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202166.108723:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202166.108729:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202166.109376:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202166.109378:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202166.109571:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202166.109572:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202166.109745:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202166.109747:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202166.110040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202166.110041:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202171.228901:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202171.228908:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202171.229655:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202171.229657:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202171.229846:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202171.229849:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202171.230053:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202171.230055:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202171.230324:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202171.230326:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202176.348969:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202176.349509:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202176.349757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202176.349958:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202176.350274:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202181.468817:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202181.468825:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202181.469549:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202181.469552:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202181.469846:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202181.469849:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202181.470142:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202181.470144:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202181.470387:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202181.470390:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202186.588773:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202186.588779:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202186.589308:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202186.589310:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202186.589558:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202186.589561:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202186.589798:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202186.589800:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202186.590022:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202186.590023:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202191.708717:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202191.708721:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202191.709260:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202191.709262:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202191.709458:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202191.709461:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202191.709666:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202191.709668:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202191.709818:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202191.709820:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202196.828817:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202196.828822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202196.829834:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202196.829837:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202196.830113:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202196.830115:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202196.830318:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202196.830319:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202196.830511:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202196.830512:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202201.948796:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202201.948805:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202201.949618:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202201.949621:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202201.949924:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202201.949927:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202201.950249:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202201.950252:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202201.950677:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202201.950680:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202207.068648:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202207.068652:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202207.069328:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202207.069329:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202207.069521:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202207.069523:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202207.069731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202207.069734:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202207.070063:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202207.070065:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788202212.188898:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202212.188905:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202212.189584:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202212.189585:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202212.189754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202212.189756:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202212.189950:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202212.189952:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202212.190200:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202212.190203:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202217.308774:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202217.308780:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202217.309399:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202217.309401:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202217.309599:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202217.309600:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202217.309772:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202217.309774:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202217.309976:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202217.309978:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202222.428926:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202222.428935:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202222.430011:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202222.430013:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202222.430212:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202222.430213:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202222.430425:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202222.430427:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202222.430686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202222.430687:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202227.548922:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202227.548934:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788202227.549826:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202227.549829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788202227.550074:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202227.550076:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788202227.550298:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202227.550300:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788202227.550462:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202227.550464:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788202232.668826:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202232.668833:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788202232.669431:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202232.669433:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788202232.669675:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202232.669677:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788202232.669950:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202232.669952:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788202232.670209:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202232.670211:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000080:00000010:3.1:1788202233.506122:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf00. 00010000:00000010:0.1:1788202233.506135:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858800. 00000080:00000010:2.1:1788202233.506145:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbd80. 00000080:00000010:2.1:1788202233.506159:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441900. 00000080:00000010:2.1:1788202233.506165:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db580. 00010000:00000010:1.1:1788202233.509123:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f800. 00000080:00000010:2.1:1788202233.509125:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db6c0. 00010000:00000010:1.1:1788202233.509131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1600. 00010000:00000010:0.1:1788202233.509138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646a00. 00010000:00000010:1.1:1788202233.509140:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737d000. 00000080:00000010:2.1:1788202233.509140:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d140. 00010000:00000010:1.1:1788202233.509150:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35600. 00010000:00000010:1.1:1788202233.509156:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858100. 00000080:00000010:2.1:1788202233.509156:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc120. 00010000:00000010:1.1:1788202233.509166:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90646600. 00000080:00000010:2.1:1788202233.509170:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad0fc040. 00000080:00000010:2.1:1788202233.509178:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d320. 00000080:00000010:2.1:1788202233.509182:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db5e0. 00010000:00000010:2.1:1788202233.509185:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d000. 00010000:00000010:2.1:1788202233.509188:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c100. 00010000:00000010:2.1:1788202233.509191:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045600. 00010000:00000010:2.1:1788202233.509195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb3858200. 00010000:00000010:2.1:1788202233.509197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647e00. 00010000:00000010:2.1:1788202233.509199:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044e00. 00010000:00000010:2.1:1788202233.509202:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3ca00. 00010000:00000010:2.1:1788202233.509203:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2044800. 00010000:00000010:2.1:1788202233.509205:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df1600. 00010000:00000010:2.1:1788202233.509208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0c00. 00010000:00000010:2.1:1788202233.509210:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c600. 00010000:00000010:2.1:1788202233.509212:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0400. 00010000:00000010:2.1:1788202233.509214:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35a00. 00010000:00000010:2.1:1788202233.509216:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35000. 00010000:00000010:2.1:1788202233.509218:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c100. 00010000:00000010:2.1:1788202233.509220:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c35c00. 00010000:00000010:2.1:1788202233.509221:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7b90647800. 00010000:00000010:2.1:1788202233.509223:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737de00. 00010000:00000010:2.1:1788202233.509226:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187ea00. 00010000:00000010:2.1:1788202233.509228:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb4c4a100. 00010000:00000010:2.1:1788202233.509230:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187e000. 00010000:00000010:2.1:1788202233.509232:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7aa737db00. 00010000:00000010:2.1:1788202233.509233:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7ab6c34400. 00010000:00000010:2.1:1788202233.509236:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bbd91c500. 00010000:00000010:2.1:1788202233.509237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7a87df0600. 00000800:00000200:1.1:1788202237.788768:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202237.788776:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202237.789470:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202237.789475:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202237.789765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202237.789768:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202237.790038:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202237.790040:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202237.790322:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202237.790324:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788202242.908692:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202242.909063:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202242.909444:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202242.909599:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202242.909760:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202248.028922:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202248.028928:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202248.029434:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202248.029437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202248.029817:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202248.029819:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202248.030117:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202248.030120:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202248.030326:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202248.030327:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202253.148731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202253.148737:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202253.149395:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202253.149398:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202253.149618:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202253.149619:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202253.149826:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202253.149827:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202253.150012:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202253.150014:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000080:00000010:2.1:1788202253.986133:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db640. 00000080:00000010:2.1:1788202253.986153:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d400. 00000080:00000010:2.1:1788202253.986158:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbee0. 00000080:00000010:2.1:1788202253.986163:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db480. 00010000:00000010:2.1:1788202253.990119:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84800. 00010000:00000010:2.1:1788202253.990131:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9cc00. 00010000:00000010:2.1:1788202253.990135:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19ae00. 00010000:00000010:2.1:1788202253.990138:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b600. 00010000:00000010:2.1:1788202253.990142:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86f00. 00010000:00000010:2.1:1788202253.990145:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a400. 00010000:00000010:2.1:1788202253.990149:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8200. 00010000:00000010:2.1:1788202253.990159:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84400. 00010000:00000010:2.1:1788202253.990164:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9de00. 00010000:00000010:2.1:1788202253.990168:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9200. 00010000:00000010:2.1:1788202253.990172:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bc217b900. 00010000:00000010:2.1:1788202253.990175:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8400. 00010000:00000010:2.1:1788202253.990177:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9800. 00010000:00000010:2.1:1788202253.990180:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8c00. 00010000:00000010:2.1:1788202253.990182:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c600. 00010000:00000010:2.1:1788202253.990184:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e200. 00010000:00000010:2.1:1788202253.990187:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3cc00. 00010000:00000010:2.1:1788202253.990195:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1fc00. 00010000:00000010:2.1:1788202253.990197:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1f200. 00010000:00000010:2.1:1788202253.990201:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9dc00. 00010000:00000010:2.1:1788202253.990202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1fa00. 00010000:00000010:2.1:1788202253.990204:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9ca00. 00010000:00000010:2.1:1788202253.990206:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c800. 00010000:00000010:2.1:1788202253.990208:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b89c3c400. 00010000:00000010:2.1:1788202253.990210:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc2045c00. 00000800:00000200:1.1:1788202258.268764:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202258.268769:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202258.269195:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202258.269196:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202258.269399:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202258.269401:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202258.269561:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202258.269562:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202258.269710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202258.269712:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202263.388732:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202263.388738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202263.389484:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202263.389487:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202263.389759:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202263.389761:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202263.390003:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202263.390005:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202263.390302:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202263.390304:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202268.508685:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202268.508691:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202268.509457:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202268.509460:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202268.509776:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202268.509779:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202268.510086:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202268.510088:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202268.510519:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202268.510521:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202273.628728:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202273.628736:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202273.629473:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202273.629478:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202273.629700:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202273.629703:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202273.629925:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202273.629926:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202273.630281:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202273.630283:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202278.748770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202278.748775:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202278.749533:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202278.749538:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202278.749818:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202278.749820:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202278.750107:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202278.750109:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202278.750294:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202278.750296:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202283.868761:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202283.868766:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202283.869510:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202283.869514:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202283.869726:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202283.869728:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202283.869976:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202283.869977:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202283.870294:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202283.870296:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202288.988797:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202288.988802:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202288.989532:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202288.989535:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202288.989918:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202288.989921:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202288.990350:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202288.990353:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202288.990661:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202288.990663:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000080:00000010:1.1:1788202293.923147:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3de80. 00000080:00000010:1.1:1788202293.923194:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db400. 00010000:00000010:2.1:1788202293.927137:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b400. 00010000:00000010:2.1:1788202293.927152:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2fe00. 00010000:00000010:2.1:1788202293.927160:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bc21d7000. 00010000:00000010:2.1:1788202293.927167:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9ee00. 00010000:00000010:2.1:1788202293.927174:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e1e00. 00010000:00000010:2.1:1788202293.927180:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7bb6d2f900. 00000080:00000010:1.1:1788202293.927183:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbb40. 00010000:00000010:2.1:1788202293.927186:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bad3e0600. 00010000:00000010:2.1:1788202293.927195:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b9863c700. 00010000:00000010:2.1:1788202293.927202:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19a600. 00010000:00000010:2.1:1788202293.927208:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aaaf9fc00. 00010000:00000010:2.1:1788202293.927215:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86e00. 00010000:00000010:2.1:1788202293.927222:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bbb19b200. 00010000:00000010:2.1:1788202293.927229:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84b00. 00010000:00000010:2.1:1788202293.927237:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa187f600. 00000080:00000010:1.1:1788202293.927238:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbf20. 00010000:00000010:2.1:1788202293.927245:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86200. 00010000:00000010:2.1:1788202293.927254:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd9a00. 00010000:00000010:2.1:1788202293.927261:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84500. 00010000:00000010:2.1:1788202293.927268:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9c200. 00000080:00000010:1.1:1788202293.927274:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3da60. 00010000:00000010:2.1:1788202293.927276:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b98e84200. 00010000:00000010:2.1:1788202293.927284:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bacc9d600. 00010000:00000010:2.1:1788202293.927291:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b92a86a00. 00010000:00000010:2.1:1788202293.927299:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7bb6fd8600. 00010000:00000010:2.1:1788202293.927306:0:0:0:(ldlm_resource.c:1555:__ldlm_resource_free()) slab-freed '(res)': 256 at ffff9c7b91fa4200. 00000080:00000010:1.1:1788202293.927308:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbec0. 00010000:00000010:2.1:1788202293.927322:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff9c7aa1a1e800. 00000080:00000010:1.1:1788202293.927336:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d880. 00000080:00000010:1.1:1788202293.927362:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db7a0. 00000080:00000010:1.1:1788202293.927385:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db800. 00000080:00000010:1.1:1788202293.927397:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbae0. 00000080:00000010:1.1:1788202293.927418:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3dbfc0. 00000080:00000010:1.1:1788202293.927437:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d260. 00000080:00000010:1.1:1788202293.927442:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa441420. 00000080:00000010:1.1:1788202293.927467:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7aaa4414e0. 00000080:00000010:1.1:1788202293.927483:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d120. 00000080:00000010:1.1:1788202293.927497:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db820. 00000080:00000010:1.1:1788202293.927514:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db560. 00000080:00000010:1.1:1788202293.927530:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3d160. 00000080:00000010:1.1:1788202293.927594:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7a99f3dec0. 00000080:00000010:1.1:1788202293.927596:0:0:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff9c7bad3db6a0. 00000800:00000200:1.1:1788202294.108788:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202294.108791:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202294.109417:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202294.109419:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202294.109641:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202294.109643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202294.109846:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202294.109848:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202294.110114:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202294.110116:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202299.228805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202299.228811:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202299.229528:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202299.229533:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202299.229967:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202299.229970:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202299.230365:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202299.230368:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202299.230631:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202299.230632:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202304.348876:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202304.348881:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202304.349621:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202304.349624:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202304.349904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202304.349906:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202304.350112:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202304.350113:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202304.350392:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202304.350394:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202309.468833:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202309.469239:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202309.469642:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202309.469930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202309.470154:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202314.589021:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202314.589025:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202314.589799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202314.589803:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202314.590234:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202314.590237:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202314.590548:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202314.590551:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202314.590842:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202314.590846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202319.708593:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202319.708597:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202319.708987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202319.708989:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202319.709223:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202319.709224:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202319.709435:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202319.709437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202319.709607:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202319.709609:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202324.829020:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202324.829029:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202324.829665:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202324.829668:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202324.829994:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202324.829997:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202324.830389:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202324.830392:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202324.830710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202324.830712:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202329.948833:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202329.948838:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202329.949555:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202329.949558:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202329.949856:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202329.949858:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202329.950164:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202329.950166:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202329.950446:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202329.950449:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202335.068687:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202335.068693:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202335.069428:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202335.069429:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202335.069629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202335.069630:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202335.069810:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202335.069812:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202335.070077:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202335.070079:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202340.188736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202340.188742:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202340.189512:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202340.189515:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202340.189836:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202340.189839:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202340.190192:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202340.190195:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202340.190460:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202340.190462:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202345.308810:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202345.308818:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202345.309534:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202345.309537:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202345.309867:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202345.309870:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202345.310203:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202345.310206:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202345.310537:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202345.310540:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202350.428919:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202350.428932:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1788202350.429682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202350.429684:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1788202350.429904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202350.429907:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1788202350.430111:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202350.430113:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1788202350.430372:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202350.430374:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1788202355.548874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202355.548881:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202355.549587:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202355.549590:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202355.549939:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202355.549942:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202355.550294:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202355.550296:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202355.550612:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202355.550615:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202360.668821:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202360.668828:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202360.669601:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202360.669604:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202360.669808:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202360.669810:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202360.669993:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202360.669994:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202360.670214:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202360.670218:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202365.788885:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202365.788893:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202365.789550:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202365.789553:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202365.789875:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202365.789878:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202365.790213:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202365.790216:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202365.790543:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202365.790545:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202370.908869:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202370.908875:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202370.909639:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202370.909642:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202370.909954:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202370.909957:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202370.910292:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202370.910295:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202370.910613:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202370.910616:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202376.028763:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202376.029190:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202376.029619:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202376.029855:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202376.030031:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202381.149720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202381.149724:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202381.150168:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202381.150171:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202381.150519:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202381.150522:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202381.150853:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202381.150857:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202381.151202:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202381.151205:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788202386.268636:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202386.268640:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202386.269030:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202386.269031:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202386.269268:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202386.269270:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202386.269452:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202386.269454:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202386.269634:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202386.269636:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788202391.388827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202391.388837:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202391.389410:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202391.389412:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202391.389640:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202391.389642:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202391.389830:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202391.389831:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202391.390204:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202391.390207:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202396.508786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202396.508793:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202396.509375:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202396.509377:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202396.509648:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202396.509651:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202396.509911:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202396.509913:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788202396.510182:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202396.510185:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202401.628890:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202401.628894:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202401.629651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202401.629653:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202401.629850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202401.629851:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202401.630085:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202401.630089:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788202401.630549:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202401.630552:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788202406.748711:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202406.748718:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202406.749426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202406.749428:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202406.749651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202406.749653:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202406.749853:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202406.749854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202406.750198:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202406.750200:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788202411.868991:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202411.869007:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202411.869944:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202411.869946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202411.870233:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202411.870235:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202411.870461:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202411.870463:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202411.870649:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202411.870651:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788202416.988722:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202416.988725:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202416.989503:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202416.989506:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202416.989769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202416.989771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202416.989971:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202416.989973:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202416.990181:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202416.990183:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202422.108878:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202422.108886:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202422.109661:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202422.109665:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202422.110012:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202422.110015:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202422.110242:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202422.110243:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202422.110434:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202422.110436:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202427.228723:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202427.228728:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202427.229318:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202427.229320:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202427.229503:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202427.229505:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202427.229658:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202427.229660:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202427.229807:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202427.229808:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788202432.348677:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202432.348680:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202432.349248:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202432.349252:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202432.349645:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202432.349647:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202432.349844:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202432.349846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202432.350000:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202432.350001:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788202437.468903:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202437.468909:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202437.469426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202437.469428:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202437.469612:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202437.469614:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202437.469790:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202437.469792:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788202437.469999:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202437.470001:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202442.588667:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202442.588895:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202442.589049:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202442.589253:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202442.589385:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202447.708820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202447.708827:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202447.709711:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202447.709714:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202447.710019:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202447.710021:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202447.710394:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202447.710397:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202447.710698:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202447.710701:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202452.828854:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202452.828861:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202452.829716:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202452.829719:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202452.830050:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202452.830053:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202452.830367:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202452.830370:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202452.830775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202452.830778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202457.948778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202457.948785:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202457.949433:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202457.949436:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202457.949736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202457.949739:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202457.950062:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202457.950065:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202457.950384:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202457.950387:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788202463.069165:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202463.069177:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202463.070245:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202463.070248:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202463.070596:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202463.070599:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202463.070959:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202463.070962:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202463.071313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202463.071317:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788202468.188904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202468.188912:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202468.189718:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202468.189720:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202468.189917:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202468.189918:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202468.190109:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202468.190110:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202468.190466:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202468.190469:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788202473.308837:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202473.308846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202473.309529:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202473.309533:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202473.309984:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202473.309986:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202473.310307:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202473.310309:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202473.310571:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202473.310573:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788202478.428744:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202478.428749:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202478.429313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202478.429316:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202478.429502:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202478.429504:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202478.429720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202478.429722:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202478.429905:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202478.429906:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788202483.548640:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202483.548644:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202483.549185:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202483.549186:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202483.549368:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202483.549370:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202483.549624:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202483.549626:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202483.549864:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202483.549865:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788202488.668777:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202488.668784:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202488.669282:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202488.669283:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202488.669504:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202488.669506:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202488.669706:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202488.669707:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788202488.669882:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788202488.669885:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202493.788776:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202493.788782:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202493.789334:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202493.789335:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202493.789506:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202493.789507:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202493.789706:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202493.789707:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202493.789981:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202493.789983:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202498.908868:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202498.908877:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202498.909543:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202498.909546:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202498.909763:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202498.909765:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202498.909948:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202498.909949:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202498.910235:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202498.910237:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202504.028905:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202504.028911:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202504.029665:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202504.029667:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202504.029923:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202504.029927:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202504.030217:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202504.030220:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202504.030483:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202504.030485:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202509.149253:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202509.152037:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202509.152189:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202509.152306:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202509.152490:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202514.268953:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202514.268959:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202514.269746:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202514.269748:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202514.270026:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202514.270029:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202514.270325:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202514.270328:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202514.270584:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202514.270585:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202519.388689:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202519.388693:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202519.389260:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202519.389264:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202519.389496:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202519.389498:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202519.389664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202519.389666:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202519.389894:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202519.389896:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202524.508720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202524.508747:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202524.509352:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202524.509355:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202524.509583:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202524.509585:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202524.509824:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202524.509825:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202524.510002:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202524.510005:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202529.628671:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202529.628677:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202529.629244:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202529.629246:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202529.629451:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202529.629453:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202529.629651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202529.629653:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202529.629878:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202529.629881:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202534.748756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202534.748761:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202534.749241:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202534.749243:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202534.749403:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202534.749405:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202534.749569:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202534.749571:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202534.749747:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202534.749749:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202539.868983:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202539.868990:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202539.869632:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202539.869634:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202539.869839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202539.869841:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202539.870023:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202539.870025:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202539.870287:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202539.870289:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202544.989014:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202544.989022:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202544.990024:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202544.990026:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202544.990325:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202544.990326:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202544.990606:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202544.990609:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202544.990904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202544.990907:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202550.109214:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202550.109221:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202550.110051:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202550.110053:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202550.110275:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202550.110293:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202550.110489:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202550.110491:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202550.110754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202550.110756:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202555.228815:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202555.228820:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202555.229501:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202555.229504:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202555.229829:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202555.229832:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202555.230183:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202555.230186:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202555.230503:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202555.230506:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202560.348770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202560.348776:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202560.349406:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202560.349410:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202560.349820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202560.349821:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202560.350052:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202560.350053:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202560.350271:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202560.350273:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202565.468739:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202565.468746:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202565.469406:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202565.469408:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202565.469606:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202565.469608:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202565.469811:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202565.469813:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202565.470035:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202565.470037:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202570.588822:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202570.588830:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202570.589373:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202570.589375:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202570.589600:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202570.589601:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202570.589779:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202570.589781:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202570.589978:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202570.589980:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202575.708735:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202575.709159:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202575.709309:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202575.709462:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202575.709614:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202580.828921:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202580.828927:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202580.829971:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202580.829974:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202580.830322:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202580.830325:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202580.830649:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202580.830652:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202580.830983:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202580.830986:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202585.949067:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202585.949078:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202585.950243:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202585.950246:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202585.950561:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202585.950564:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202585.950859:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202585.950861:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202585.951138:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202585.951140:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202591.068816:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202591.068821:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202591.069280:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202591.069282:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202591.069489:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202591.069490:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202591.069748:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202591.069750:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202591.069935:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202591.069937:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202596.188774:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202596.188781:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202596.189425:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202596.189427:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202596.189656:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202596.189658:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202596.189883:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202596.189885:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202596.190139:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202596.190141:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202601.308950:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202601.308955:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202601.309567:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202601.309570:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202601.309827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202601.309829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202601.310057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202601.310060:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202601.310271:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202601.310273:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788202606.428873:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202606.428881:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202606.429641:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202606.429643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202606.429975:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202606.429977:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202606.430180:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202606.430182:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202606.430547:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202606.430550:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788202611.548890:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202611.548900:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202611.550559:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202611.550561:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202611.550749:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202611.550751:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202611.550930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202611.550932:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202611.551120:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202611.551122:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788202616.668733:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202616.668738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202616.669334:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202616.669336:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202616.669557:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202616.669559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202616.669794:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202616.669796:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202616.669977:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202616.669979:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788202621.788961:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202621.788967:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202621.790141:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202621.790143:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202621.790363:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202621.790366:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202621.790583:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202621.790585:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202621.790880:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202621.790882:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788202626.908976:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202626.908985:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202626.909616:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202626.909619:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202626.909967:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202626.909970:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202626.910336:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202626.910339:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202626.910681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202626.910684:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788202632.028760:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202632.028765:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202632.029350:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202632.029353:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202632.029558:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202632.029560:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202632.029740:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202632.029741:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202632.029978:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202632.029980:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788202637.148800:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202637.148807:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202637.149452:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202637.149454:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202637.149638:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202637.149640:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202637.149862:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202637.149863:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202637.150044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202637.150046:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788202642.268772:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202642.269239:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202642.269533:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202642.269713:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202642.269965:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202647.388939:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202647.388946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202647.389782:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202647.389785:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202647.390059:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202647.390061:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202647.390288:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202647.390290:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202647.390543:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202647.390545:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202652.508710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202652.508714:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202652.509334:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202652.509336:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202652.509607:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202652.509608:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202652.509819:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202652.509820:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202652.510048:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202652.510050:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788202657.628916:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202657.628923:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202657.630043:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202657.630045:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202657.630549:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202657.630553:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202657.630960:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202657.630964:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202657.631494:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202657.631496:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788202662.748756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202662.748770:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202662.749468:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202662.749470:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202662.749655:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202662.749657:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202662.749855:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202662.749857:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202662.750143:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202662.750145:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202667.868789:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202667.868794:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202667.869499:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202667.869501:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202667.869724:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202667.869726:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202667.869965:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202667.869967:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202667.870313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202667.870315:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202672.988813:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202672.988820:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202672.989480:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202672.989483:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202672.989795:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202672.989797:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202672.990111:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202672.990113:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202672.990419:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202672.990421:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202678.108870:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202678.108877:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202678.109453:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202678.109456:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202678.109704:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202678.109706:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202678.109944:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202678.109946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202678.110169:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202678.110172:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202683.228825:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202683.228833:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202683.229492:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202683.229494:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202683.229724:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202683.229726:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202683.229915:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202683.229918:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202683.230175:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202683.230176:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202688.348765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202688.348774:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202688.349496:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202688.349498:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202688.349699:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202688.349701:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202688.349949:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202688.349951:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202688.350219:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202688.350221:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202693.468720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202693.468725:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202693.469382:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202693.469384:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202693.469681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202693.469683:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202693.470034:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202693.470037:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202693.470485:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202693.470488:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202698.588841:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202698.588849:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202698.589547:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202698.589550:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202698.589897:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202698.589901:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202698.590250:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202698.590253:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202698.590579:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202698.590582:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788202703.708784:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202703.708791:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788202703.709464:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202703.709467:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788202703.709797:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202703.709800:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788202703.710115:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202703.710118:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788202703.710439:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202703.710441:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788202708.828741:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202708.829172:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202708.829335:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202708.829473:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202708.829694:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202713.949289:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202713.949293:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202713.949657:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202713.949659:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202713.949945:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202713.949947:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202713.950238:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202713.950241:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202713.950491:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202713.950493:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788202719.068769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202719.068775:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202719.069556:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202719.069559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202719.069778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202719.069780:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202719.070000:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202719.070003:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202719.070360:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202719.070363:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202724.188806:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202724.188812:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202724.189498:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202724.189499:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202724.189693:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202724.189694:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202724.189871:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202724.189873:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202724.190133:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202724.190134:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202729.308799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202729.308805:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788202729.309554:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202729.309558:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788202729.309934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202729.309936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788202729.310256:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202729.310258:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788202729.310514:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202729.310517:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202734.428872:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202734.428878:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202734.429492:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202734.429495:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202734.429824:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202734.429826:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202734.430015:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202734.430016:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202734.430256:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202734.430258:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202739.548819:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202739.548825:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202739.549370:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202739.549374:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202739.549641:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202739.549643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202739.549946:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202739.549949:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202739.550313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202739.550316:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202744.668790:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202744.668794:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202744.669413:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202744.669415:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202744.669610:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202744.669613:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202744.669888:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202744.669890:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202744.670217:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202744.670220:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202749.788742:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202749.788748:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202749.789563:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202749.789566:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202749.789805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202749.789807:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202749.790053:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202749.790055:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202749.790370:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202749.790372:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202754.908848:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202754.908854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202754.909434:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202754.909436:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202754.909626:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202754.909628:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202754.909849:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202754.909850:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202754.910060:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202754.910062:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202760.028987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202760.028991:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202760.029360:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202760.029363:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202760.029587:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202760.029589:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202760.029794:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202760.029796:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202760.029999:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202760.030000:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202765.148837:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202765.148840:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202765.150313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202765.150319:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202765.151030:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202765.151032:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202765.151246:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202765.151248:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202765.151426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202765.151427:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202770.269010:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202770.269030:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202770.269849:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202770.269851:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202770.270181:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202770.270184:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202770.270507:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202770.270510:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202770.270815:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202770.270817:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202775.388803:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202775.389256:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202775.389432:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202775.389596:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202775.389745:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202780.509704:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202780.509709:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202780.510077:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202780.510080:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202780.510444:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202780.510447:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202780.510750:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202780.510752:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202780.511021:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202780.511022:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:0.1:1788202785.628831:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202785.628836:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202785.629512:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202785.629515:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202785.629736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202785.629738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202785.629913:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202785.629915:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202785.630269:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202785.630271:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788202790.748738:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202790.748744:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202790.749372:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202790.749374:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202790.749600:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202790.749617:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202790.749836:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202790.749838:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202790.750155:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202790.750158:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788202795.868674:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202795.868679:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202795.869417:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202795.869418:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202795.869700:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202795.869702:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202795.870036:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202795.870038:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202795.870385:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202795.870388:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788202800.988830:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202800.988839:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202800.989501:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202800.989504:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202800.989830:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202800.989832:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202800.990184:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202800.990187:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202800.990483:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202800.990485:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202806.109048:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202806.109056:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202806.110768:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202806.110772:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202806.111117:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202806.111120:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202806.111414:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202806.111417:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202806.111702:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202806.111704:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202811.228691:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202811.228697:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202811.229438:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202811.229440:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202811.229639:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202811.229641:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202811.229839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202811.229841:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202811.230207:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202811.230209:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202816.348808:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202816.348815:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202816.349370:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202816.349372:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202816.349564:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202816.349565:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202816.349809:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202816.349812:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202816.350040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202816.350043:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202821.469069:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202821.469080:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202821.470219:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202821.470222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202821.470527:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202821.470530:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202821.470848:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202821.470851:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202821.471166:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202821.471169:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202826.588926:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202826.588933:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202826.589663:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202826.589665:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202826.589926:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202826.589929:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202826.590185:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202826.590188:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202826.590410:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202826.590412:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788202831.708714:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202831.708718:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202831.709459:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202831.709460:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202831.709685:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202831.709686:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202831.709863:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202831.709865:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202831.710182:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202831.710184:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788202836.829032:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202836.829042:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202836.829904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202836.829906:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202836.830219:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202836.830222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202836.830505:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202836.830507:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202836.830702:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202836.830705:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788202841.949003:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202841.949425:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202841.949598:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202841.949740:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202841.949922:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202847.069499:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202847.069503:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202847.069917:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202847.069920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202847.070302:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202847.070305:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202847.070681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202847.070683:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202847.070906:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202847.070908:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202852.188770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202852.188775:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202852.189258:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202852.189260:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202852.189468:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202852.189470:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202852.189696:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202852.189697:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202852.189880:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202852.189882:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202857.308716:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202857.308721:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202857.309425:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202857.309429:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202857.309876:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202857.309879:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202857.310324:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202857.310327:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202857.310641:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202857.310643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202862.428925:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202862.428934:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202862.429745:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202862.429748:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202862.430227:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202862.430229:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202862.430462:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202862.430465:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202862.430765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202862.430767:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202867.548891:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202867.548898:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202867.549600:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202867.549602:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202867.549815:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202867.549817:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202867.550023:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202867.550025:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202867.550282:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202867.550284:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788202872.668756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202872.668761:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202872.669461:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202872.669464:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202872.669709:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202872.669711:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202872.669893:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202872.669895:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202872.670208:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202872.670210:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788202877.788836:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202877.788843:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202877.789407:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202877.789409:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202877.789621:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202877.789623:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202877.789824:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202877.789825:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202877.790001:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202877.790002:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788202882.908902:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202882.908910:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202882.909525:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202882.909527:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202882.909831:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202882.909834:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202882.910168:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202882.910171:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202882.910477:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202882.910479:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788202888.028881:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202888.028888:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202888.029468:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202888.029470:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202888.029758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202888.029760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202888.030050:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202888.030053:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202888.030364:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202888.030366:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202893.148825:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202893.148829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202893.149512:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202893.149515:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202893.149695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202893.149697:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202893.149890:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202893.149891:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202893.150170:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202893.150172:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202898.268817:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202898.268822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202898.269753:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202898.269756:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202898.270068:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202898.270071:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202898.270415:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202898.270417:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202898.270716:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202898.270717:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202903.388851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202903.388858:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202903.389373:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202903.389375:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202903.389556:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202903.389559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202903.389852:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202903.389855:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202903.390181:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202903.390184:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202908.509125:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202908.509683:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202908.509858:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202908.510010:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202908.510425:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202913.629028:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202913.629040:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202913.630759:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202913.630762:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202913.631257:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202913.631260:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202913.631580:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202913.631584:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202913.631828:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202913.631830:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202918.748660:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202918.748666:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202918.749248:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202918.749251:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202918.749511:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202918.749513:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202918.749672:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202918.749674:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202918.749855:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202918.749857:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202923.868786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202923.868801:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202923.869447:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202923.869450:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202923.869660:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202923.869662:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202923.869843:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202923.869845:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202923.870075:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202923.870077:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788202928.988765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202928.988771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202928.989529:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202928.989532:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202928.989736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202928.989737:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202928.989905:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202928.989907:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202928.990191:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202928.990194:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788202934.108572:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202934.108576:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202934.108997:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202934.108999:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202934.109227:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202934.109229:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202934.109476:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202934.109479:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202934.109722:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202934.109723:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788202939.228718:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202939.228722:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788202939.229469:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202939.229472:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788202939.229806:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202939.229809:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788202939.230128:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202939.230130:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788202939.230404:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202939.230407:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788202944.348671:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202944.348677:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202944.349482:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202944.349485:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202944.349732:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202944.349734:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202944.349944:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202944.349946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202944.350316:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202944.350318:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788202949.468986:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202949.468991:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202949.469909:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202949.469915:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202949.470455:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202949.470457:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202949.470714:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202949.470717:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202949.470948:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202949.470950:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788202954.588791:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202954.588797:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202954.589348:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202954.589350:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202954.589540:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202954.589541:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202954.589758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202954.589759:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202954.589934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202954.589936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788202959.708850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202959.708858:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202959.709591:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202959.709594:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202959.709900:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202959.709903:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202959.710251:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202959.710254:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202959.710563:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202959.710566:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788202964.828811:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202964.828817:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202964.829566:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202964.829569:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202964.829910:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202964.829913:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202964.830291:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202964.830294:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202964.830630:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202964.830633:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788202969.948768:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202969.948774:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202969.949399:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202969.949400:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202969.949622:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202969.949624:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202969.949804:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202969.949806:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202969.950067:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202969.950069:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788202975.068710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202975.069076:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202975.069493:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202975.069716:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202975.069859:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202980.188931:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202980.188938:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202980.189535:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202980.189537:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202980.190129:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202980.190132:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202980.190532:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202980.190535:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202980.190839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202980.190842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788202985.308801:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202985.308807:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202985.309477:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202985.309479:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202985.309757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202985.309760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202985.310045:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202985.310047:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788202985.310282:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788202985.310284:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788202990.428764:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202990.428806:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202990.429449:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202990.429454:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202990.429770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202990.429772:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202990.430021:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202990.430024:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202990.430432:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202990.430435:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788202995.549195:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202995.549207:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202995.550192:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202995.550195:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202995.550501:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202995.550504:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202995.550827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202995.550830:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788202995.551167:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788202995.551170:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203000.668805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203000.668812:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203000.669522:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203000.669526:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203000.669786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203000.669787:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203000.669967:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203000.669969:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203000.670306:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203000.670308:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203005.788761:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203005.788767:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203005.789418:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203005.789420:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203005.789663:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203005.789666:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203005.789875:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203005.789877:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203005.790152:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203005.790155:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203010.908939:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203010.908944:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203010.909700:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203010.909702:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203010.909969:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203010.909971:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203010.910249:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203010.910251:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203010.910532:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203010.910534:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203016.028738:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203016.028744:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203016.029433:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203016.029436:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203016.029771:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203016.029774:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203016.030040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203016.030042:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203016.030367:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203016.030369:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203021.148922:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203021.148930:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203021.149766:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203021.149771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203021.150082:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203021.150084:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203021.150417:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203021.150420:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203021.150603:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203021.150605:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203026.269118:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203026.269123:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203026.269933:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203026.269935:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203026.270181:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203026.270183:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203026.270459:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203026.270462:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203026.270723:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203026.270725:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203031.388904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203031.388912:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203031.389519:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203031.389522:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203031.389837:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203031.389841:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203031.390191:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203031.390194:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203031.390500:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203031.390503:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203036.508720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203036.508728:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203036.509327:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203036.509329:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203036.509521:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203036.509522:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203036.509710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203036.509712:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203036.509975:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203036.509977:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203041.629033:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203041.629637:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203041.629900:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203041.630133:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203041.630267:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203046.748747:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203046.748754:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203046.749475:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203046.749478:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203046.749758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203046.749760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203046.750010:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203046.750012:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203046.750349:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203046.750352:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203051.868717:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203051.868721:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203051.869481:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203051.869483:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203051.869668:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203051.869669:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203051.869852:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203051.869854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203051.870069:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203051.870071:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203056.988761:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203056.988769:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203056.989447:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203056.989449:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203056.989682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203056.989684:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203056.989934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203056.989936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203056.990213:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203056.990215:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203062.108774:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203062.108778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203062.109514:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203062.109516:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203062.109765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203062.109767:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203062.109952:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203062.109955:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203062.110251:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203062.110253:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203067.228814:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203067.228820:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203067.229436:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203067.229439:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203067.229848:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203067.229849:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203067.230105:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203067.230107:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203067.230288:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203067.230289:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203072.348787:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203072.348793:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203072.349469:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203072.349470:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203072.349755:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203072.349757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203072.349987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203072.349989:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203072.350312:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203072.350314:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203077.468938:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203077.468945:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203077.469843:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203077.469845:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203077.470049:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203077.470052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203077.470304:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203077.470306:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203077.470486:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203077.470487:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203082.588764:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203082.588772:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203082.589533:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203082.589536:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203082.589870:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203082.589873:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203082.590236:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203082.590239:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203082.590559:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203082.590562:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203087.708894:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203087.708901:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203087.709710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203087.709713:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203087.710049:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203087.710052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203087.710389:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203087.710392:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203087.710690:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203087.710693:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203092.828742:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203092.828749:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203092.829443:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203092.829446:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203092.829681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203092.829683:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203092.829914:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203092.829916:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203092.830228:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203092.830230:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203097.948756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203097.948761:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203097.949287:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203097.949289:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203097.949492:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203097.949495:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203097.949681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203097.949683:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203097.949889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203097.949892:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203103.068773:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203103.068779:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203103.069415:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203103.069417:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203103.069616:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203103.069618:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203103.069927:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203103.069930:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203103.070385:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203103.070388:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203108.188859:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203108.189299:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203108.189704:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203108.189854:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203108.190464:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203113.308852:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203113.308860:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203113.309651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203113.309654:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203113.309950:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203113.309953:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203113.310280:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203113.310283:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203113.310553:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203113.310557:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203118.428909:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203118.428917:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203118.429560:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203118.429562:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203118.429757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203118.429758:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203118.429939:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203118.429940:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203118.430147:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203118.430150:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203123.548886:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203123.548901:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203123.549654:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203123.549657:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203123.549982:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203123.549985:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203123.550334:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203123.550337:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203123.550775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203123.550778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203128.669967:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203128.669983:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203128.670377:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203128.670380:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203128.670641:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203128.670643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203128.670826:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203128.670828:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203128.671081:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203128.671084:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203133.788854:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203133.788861:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203133.789438:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203133.789440:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203133.789625:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203133.789627:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203133.789805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203133.789807:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203133.790010:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203133.790011:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203138.908971:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203138.908975:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203138.909749:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203138.909750:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203138.909943:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203138.909945:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203138.910167:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203138.910169:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203138.910481:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203138.910484:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203144.028790:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203144.028794:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203144.029521:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203144.029525:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203144.029806:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203144.029807:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203144.030156:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203144.030159:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203144.030481:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203144.030483:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203149.148760:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203149.148763:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203149.149350:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203149.149352:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203149.149565:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203149.149569:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203149.149756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203149.149757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203149.150005:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203149.150007:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203154.268779:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203154.268786:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203154.269397:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203154.269399:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203154.269614:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203154.269615:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203154.269816:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203154.269818:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203154.270027:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203154.270029:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203159.388982:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203159.388991:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203159.389660:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203159.389663:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203159.390006:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203159.390008:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203159.390347:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203159.390350:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203159.390661:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203159.390664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203164.508874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203164.508878:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203164.509450:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203164.509453:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203164.509778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203164.509782:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203164.510060:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203164.510061:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203164.510289:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203164.510290:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203169.628869:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203169.628878:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203169.629556:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203169.629559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203169.629880:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203169.629883:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203169.630219:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203169.630222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203169.630561:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203169.630564:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203174.749014:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203174.749317:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203174.749430:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203174.749596:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203174.749777:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203179.868924:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203179.868932:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203179.869739:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203179.869754:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203179.870087:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203179.870122:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203179.870450:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203179.870453:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203179.870751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203179.870754:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203184.988930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203184.988936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203184.989741:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203184.989744:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203184.990054:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203184.990057:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203184.990410:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203184.990413:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203184.990839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203184.990842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203190.108618:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203190.108622:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203190.108978:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203190.108980:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203190.109200:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203190.109202:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203190.109470:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203190.109472:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203190.109745:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203190.109747:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203195.229197:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203195.229201:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203195.229525:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203195.229528:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203195.229798:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203195.229800:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203195.230075:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203195.230109:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203195.230372:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203195.230374:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203200.348754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203200.348760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203200.349316:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203200.349319:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203200.349510:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203200.349512:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203200.349671:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203200.349672:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203200.349827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203200.349829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203205.468770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203205.468778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203205.469279:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203205.469281:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203205.469491:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203205.469492:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203205.469672:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203205.469674:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203205.469881:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203205.469883:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203210.588831:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203210.588838:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203210.589323:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203210.589326:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203210.589607:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203210.589610:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203210.589898:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203210.589900:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203210.590180:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203210.590182:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203215.708743:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203215.708750:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203215.709247:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203215.709249:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203215.709443:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203215.709445:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203215.709660:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203215.709661:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203215.709919:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203215.709920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203220.828663:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203220.828667:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203220.829280:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203220.829282:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203220.829466:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203220.829468:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203220.829649:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203220.829651:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203220.829919:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203220.829920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203225.948659:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203225.948664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203225.949262:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203225.949263:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203225.949452:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203225.949453:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203225.949694:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203225.949696:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203225.949949:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203225.949951:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203231.068763:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203231.068768:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203231.069430:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203231.069432:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203231.069651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203231.069653:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203231.069814:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203231.069816:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203231.070065:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203231.070067:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203236.188682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203236.188686:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203236.189313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203236.189314:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203236.189500:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203236.189502:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203236.189658:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203236.189660:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203236.189871:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203236.189873:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203241.308971:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203241.309535:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203241.309709:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203241.309854:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203241.309999:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203246.428772:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203246.428778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203246.431255:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203246.431258:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203246.431575:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203246.431577:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203246.431764:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203246.431766:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203246.431981:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203246.431983:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203251.548800:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203251.548805:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203251.549445:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203251.549448:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203251.549651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203251.549653:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203251.549858:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203251.549861:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203251.550278:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203251.550280:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:0.1:1788203256.668786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203256.668794:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203256.669369:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203256.669370:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203256.669565:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203256.669567:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203256.669812:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203256.669814:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203256.670028:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203256.670030:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203261.788744:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203261.788750:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203261.789263:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203261.789265:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203261.789539:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203261.789541:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203261.789807:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203261.789809:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203261.790111:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203261.790114:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203266.908799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203266.908807:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203266.909546:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203266.909549:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203266.909858:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203266.909860:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203266.910161:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203266.910164:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203266.910470:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203266.910473:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203272.028825:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203272.028831:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203272.029581:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203272.029583:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203272.029946:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203272.029949:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203272.030311:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203272.030314:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203272.030653:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203272.030656:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203277.148735:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203277.148738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203277.149460:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203277.149465:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203277.149768:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203277.149770:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203277.149984:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203277.149987:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203277.150320:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203277.150322:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203282.268911:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203282.268920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203282.269522:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203282.269525:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203282.269772:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203282.269774:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203282.269956:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203282.269959:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203282.270189:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203282.270192:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203287.388772:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203287.388779:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203287.389658:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203287.389661:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203287.389920:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203287.389922:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203287.390208:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203287.390212:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203287.390444:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203287.390446:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203292.508786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203292.508795:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788203292.509548:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203292.509550:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788203292.509851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203292.509854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788203292.510185:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203292.510188:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788203292.510512:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203292.510515:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788203297.628854:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203297.628877:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203297.629488:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203297.629490:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203297.629677:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203297.629679:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203297.629952:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203297.629954:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203297.630236:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203297.630238:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203302.748837:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203302.748842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203302.750623:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203302.750625:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203302.750837:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203302.750839:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203302.751027:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203302.751029:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203302.751243:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203302.751246:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203307.868806:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203307.869315:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203307.869513:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203307.869682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203307.869843:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203312.988966:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203312.988974:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203312.989861:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203312.989864:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203312.990202:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203312.990203:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203312.990407:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203312.990409:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203312.990591:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203312.990593:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203318.108865:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203318.108870:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203318.109499:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203318.109501:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203318.109749:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203318.109751:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203318.109937:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203318.109938:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203318.110169:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203318.110171:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203323.228997:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203323.229005:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203323.229713:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203323.229716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203323.229993:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203323.229995:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203323.230243:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203323.230245:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203323.230502:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203323.230504:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203328.348698:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203328.348705:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203328.349483:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203328.349485:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203328.349695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203328.349697:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203328.349938:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203328.349941:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203328.350269:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203328.350271:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203333.468832:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203333.468840:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203333.469421:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203333.469422:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203333.469607:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203333.469608:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203333.469820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203333.469822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203333.470009:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203333.470012:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203338.588892:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203338.588900:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203338.589526:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203338.589528:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203338.589746:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203338.589748:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203338.589957:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203338.589959:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203338.590289:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203338.590290:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203343.708872:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203343.708877:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203343.709549:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203343.709551:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203343.709884:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203343.709886:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203343.710158:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203343.710160:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203343.710344:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203343.710345:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203348.828753:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203348.828760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203348.829426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203348.829428:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203348.829643:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203348.829645:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203348.829860:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203348.829862:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203348.830104:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203348.830107:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203353.948676:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203353.948680:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203353.949077:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203353.949080:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203353.949366:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203353.949369:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203353.949560:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203353.949561:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203353.949738:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203353.949740:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203359.068796:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203359.068800:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203359.069509:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203359.069512:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203359.069830:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203359.069833:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203359.070177:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203359.070180:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203359.070491:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203359.070494:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203364.188950:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203364.188957:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203364.189883:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203364.189886:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203364.190275:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203364.190278:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203364.190608:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203364.190611:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203364.191085:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203364.191114:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203369.308923:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203369.308930:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203369.309749:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203369.309752:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203369.310047:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203369.310049:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203369.310396:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203369.310399:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203369.310704:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203369.310707:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203374.428718:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203374.429164:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203374.429429:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203374.429611:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203374.429762:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203379.549209:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203379.549217:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203379.549938:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203379.549941:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203379.550370:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203379.550373:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203379.550704:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203379.550706:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203379.551218:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203379.551222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203384.668770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203384.668776:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203384.669429:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203384.669431:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203384.669616:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203384.669618:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203384.669820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203384.669822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203384.670008:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203384.670010:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203389.788796:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203389.788802:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203389.789480:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203389.789482:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203389.789731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203389.789734:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203389.789970:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203389.789971:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203389.790172:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203389.790174:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203394.908887:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203394.908893:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203394.909542:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203394.909545:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203394.909838:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203394.909841:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203394.910022:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203394.910023:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203394.910295:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203394.910297:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203400.028759:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203400.028765:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203400.029420:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203400.029422:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203400.029706:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203400.029708:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203400.029963:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203400.029965:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203400.030173:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203400.030175:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203405.149206:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203405.149212:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203405.149846:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203405.149848:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203405.150040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203405.150042:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203405.150248:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203405.150250:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788203405.150447:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203405.150450:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1788203410.268766:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203410.268776:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788203410.268779:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203410.268787:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203410.268796:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203410.268800:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203410.268805:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203410.268808:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203410.268815:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203410.268825:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203410.268837:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543299840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203410.268860:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203410.268866:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203410.268873:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543299840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203410.268933:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203410.268939:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203410.268944:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203410.268949:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203410.268951:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203410.268961:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203410.268971:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25096 00000400:00000200:2.0:1788203410.268980:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203410.268986:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203410.268990:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203410.268998:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269011:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203410.269019:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269024:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203410.269035:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203410.269040:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203410.269050:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203410.269057:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203410.269071:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203410.269081:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203410.269134:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269149:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203410.269155:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269157:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203410.269164:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203410.269176:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269181:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203410.269194:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203410.269203:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203410.269210:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203410.269213:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203410.269216:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203410.269218:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203410.269225:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203410.269231:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203410.269236:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203410.269240:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203410.269243:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203410.269249:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203410.269251:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203410.269253:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203410.269259:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203410.269263:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203410.269271:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543299968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1788203410.269271:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203410.269286:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000100:00000001:2.0:1788203410.269287:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203410.269291:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203410.269300:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543299968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203410.269321:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203410.269324:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203410.269331:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203410.269333:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203410.269334:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203410.269338:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203410.269359:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25096 00000400:00000200:2.0:1788203410.269371:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203410.269380:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203410.269383:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203410.269390:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269393:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203410.269399:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269402:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203410.269406:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203410.269409:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203410.269411:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203410.269413:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203410.269418:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203410.269421:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203410.269431:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269438:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203410.269443:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269452:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203410.269459:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203410.269464:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203410.269472:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203410.269478:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203410.269481:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203410.269484:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203410.269490:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203410.269493:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203410.269497:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203410.269503:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203410.269510:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203410.269516:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203410.269523:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203410.269527:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203410.269531:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203410.269533:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203410.269535:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203410.269544:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203410.269549:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203410.269556:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543300096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203410.269586:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203410.269592:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203410.269601:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543300096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203410.269614:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203410.269620:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25106) 00000800:00000200:0.1:1788203410.269972:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203410.269975:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203410.270328:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203410.270332:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203410.270629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203410.270632:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788203410.270844:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203410.270846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000100:00000001:2.0:1788203411.100311:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203411.100322:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203411.100326:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203411.100350:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203411.100355:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203411.100361:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203411.100370:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203411.100374:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203411.100382:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203411.612271:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319770374, at 4319770432 00000100:00000010:2.0:1788203411.612296:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa060. 00000100:00000001:2.0:1788203412.124241:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203412.124260:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203412.124266:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203412.124285:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203412.124299:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203412.124303:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203412.124309:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203412.124333:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203412.124340:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203413.148242:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203413.148252:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203413.148262:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203413.148277:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203413.148288:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203413.148305:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203413.148312:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203413.148322:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203413.148328:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203414.172199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203414.172209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203414.172215:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203414.172225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203414.172232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203414.172238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203414.172246:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203414.172250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203414.172255:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203415.196229:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203415.196241:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203415.196245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203415.196252:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203415.196270:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203415.196275:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203415.196286:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203415.196291:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203415.196293:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203415.388851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203415.388858:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788203415.389464:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203415.389466:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788203415.389650:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203415.389652:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788203415.389825:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203415.389827:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:0.1:1788203415.390162:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203415.390165:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000100:00000001:2.0:1788203416.220219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203416.220229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203416.220252:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203416.220254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203416.220257:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203416.220260:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203416.220265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203416.220269:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203416.220274:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203416.732293:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319775493, at 4319775552 00000100:00000010:2.0:1788203416.732312:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa900. 00000100:00000001:2.0:1788203417.244249:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203417.244257:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203417.244259:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203417.244273:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203417.244287:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203417.244295:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203417.244300:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203417.244308:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203417.244323:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203418.268256:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203418.268269:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203418.268275:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203418.268285:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203418.268293:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203418.268301:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203418.268306:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203418.268316:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203418.268324:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203419.292202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203419.292211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203419.292216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203419.292223:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203419.292231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203419.292235:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203419.292241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203419.292245:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203419.292250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203420.316258:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203420.316272:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203420.316277:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203420.316283:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203420.316290:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203420.316297:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203420.316302:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203420.316308:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203420.316317:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203420.508876:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203420.508881:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203420.509630:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203420.509632:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203420.509830:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203420.509832:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203420.510014:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203420.510016:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203420.510417:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203420.510421:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788203421.340245:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203421.340257:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203421.340261:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203421.340270:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203421.340278:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203421.340283:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203421.340287:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203421.340292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203421.340296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203421.852220:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319780613, at 4319780672 00000100:00000010:2.0:1788203421.852246:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa4e0. 00000100:00000001:2.0:1788203422.364280:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203422.364297:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203422.364303:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203422.364314:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203422.364323:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203422.364329:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203422.364334:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203422.364340:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203422.364347:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203423.388248:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203423.388260:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203423.388266:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203423.388276:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203423.388284:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203423.388292:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203423.388299:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203423.388306:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203423.388311:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203424.412287:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203424.412294:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203424.412296:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203424.412301:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203424.412306:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203424.412310:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203424.412313:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203424.412316:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203424.412319:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.436204:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203425.436214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203425.436218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203425.436225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203425.436231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203425.436236:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203425.436241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203425.436246:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203425.436251:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203425.628238:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203425.628250:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203425.628253:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203425.628258:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203425.628265:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203425.628273:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25112 00000400:00000200:2.0:1788203425.628282:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203425.628291:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203425.628296:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203425.628301:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628305:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.628307:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628310:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203425.628314:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203425.628317:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203425.628321:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203425.628325:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203425.628327:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203425.628331:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203425.628334:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628336:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.628339:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628341:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203425.628344:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203425.628348:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628352:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203425.628355:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203425.628358:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203425.628362:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.628364:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203425.628367:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203425.628380:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203425.628384:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203425.628388:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203425.628390:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.628393:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203425.628396:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203425.628401:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203425.628409:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203425.628416:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203425.628426:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543301504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203425.628437:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203425.628443:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203425.628448:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543301504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203425.628484:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203425.628488:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203425.628492:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203425.628493:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203425.628495:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203425.628498:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203425.628501:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25112 00000400:00000200:2.0:1788203425.628505:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203425.628509:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203425.628513:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203425.628517:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628520:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.628523:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628526:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203425.628531:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203425.628535:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203425.628537:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203425.628542:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203425.628548:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203425.628555:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203425.628564:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628571:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.628575:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628582:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203425.628590:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203425.628598:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628604:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203425.628610:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203425.628619:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203425.628625:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.628629:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203425.628636:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203425.628641:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.628643:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203425.628649:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203425.628654:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203425.628659:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203425.628663:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.628669:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203425.628672:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203425.628676:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203425.628684:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203425.628692:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203425.628702:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543301632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203425.628716:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203425.628722:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203425.628731:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543301632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203425.628752:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203425.628760:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203425.628769:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203425.628774:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203425.628777:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203425.628783:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203425.628790:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25112 00000400:00000200:2.0:1788203425.628799:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203425.628808:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203425.628815:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203425.628824:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:0.1:1788203425.628829:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203425.628830:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.628835:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628839:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:0.1:1788203425.628840:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1788203425.628845:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203425.628850:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203425.628854:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203425.628858:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203425.628864:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203425.628872:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203425.628880:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628885:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.628892:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628899:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203425.628905:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203425.628913:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203425.628919:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203425.628924:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203425.628928:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203425.628934:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.628939:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203425.628943:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203425.628949:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.628952:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203425.628956:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203425.628962:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203425.628969:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203425.628971:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.628976:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203425.628978:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203425.628980:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203425.628988:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203425.628995:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203425.629003:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543301760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203425.629017:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203425.629023:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203425.629031:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543301760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203425.629048:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203425.629062:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203425.629071:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203425.629074:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203425.629116:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203425.629121:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203425.629125:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25112 00000400:00000200:2.0:1788203425.629131:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203425.629135:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203425.629140:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203425.629148:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629155:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.629160:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629164:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203425.629169:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203425.629172:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203425.629176:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203425.629181:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203425.629185:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203425.629190:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203425.629200:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629205:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.629211:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629217:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203425.629223:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203425.629228:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629232:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203425.629236:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203425.629240:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203425.629244:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.629247:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203425.629251:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203425.629254:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.629258:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203425.629261:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203425.629264:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203425.629268:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203425.629270:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.629273:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203425.629276:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203425.629280:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203425.629285:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203425.629290:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203425.629298:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543301888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203425.629306:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203425.629308:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203425.629311:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543301888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203425.629322:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203425.629326:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203425.629331:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203425.629333:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203425.629337:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203425.629340:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203425.629344:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25112 00000400:00000200:2.0:1788203425.629352:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203425.629357:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203425.629361:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203425.629366:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629370:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.629372:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629376:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203425.629378:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203425.629380:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203425.629383:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203425.629385:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203425.629387:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203425.629391:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203425.629396:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629399:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203425.629402:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629404:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203425.629407:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203425.629410:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203425.629414:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203425.629416:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203425.629418:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203425.629422:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.629424:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203425.629427:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203425.629431:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.629434:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203425.629437:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203425.629441:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203425.629444:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203425.629446:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203425.629448:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203425.629449:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203425.629450:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203425.629452:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203425.629455:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203425.629460:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543302016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203425.629469:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203425.629472:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203425.629479:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543302016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203425.629491:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203425.629497:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25122) 00000800:00000200:0.1:1788203425.629628:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203425.629631:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203425.629951:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203425.629954:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203425.630191:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203425.630193:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203425.630448:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203425.630449:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788203426.460203:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203426.460213:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203426.460217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203426.460225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203426.460232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203426.460239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203426.460246:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203426.460250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203426.460254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203426.972206:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319785733, at 4319785792 00000100:00000010:2.0:1788203426.972224:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faba0. 00000100:00000001:2.0:1788203427.484184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203427.484192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203427.484197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203427.484203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203427.484210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203427.484216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203427.484223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203427.484227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203427.484234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203428.508168:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203428.508177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203428.508179:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203428.508186:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203428.508195:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203428.508201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203428.508208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203428.508214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203428.508222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203429.532173:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203429.532180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203429.532185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203429.532192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203429.532200:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203429.532206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203429.532213:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203429.532219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203429.532227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.556196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203430.556210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203430.556215:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203430.556228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203430.556238:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203430.556243:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203430.556247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203430.556251:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203430.556254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203430.748211:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203430.748224:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203430.748227:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203430.748231:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203430.748236:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203430.748244:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25117 00000400:00000200:2.0:1788203430.748256:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203430.748264:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203430.748267:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203430.748270:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748273:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.748275:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748277:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203430.748281:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203430.748283:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203430.748286:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203430.748289:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203430.748291:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203430.748293:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203430.748295:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748296:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.748297:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748299:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203430.748302:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203430.748305:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748307:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203430.748310:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203430.748313:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203430.748315:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.748317:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203430.748320:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203430.748330:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788203430.748332:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203430.748336:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203430.748337:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.748340:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203430.748343:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203430.748347:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203430.748355:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203430.748363:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203430.748371:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543302144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203430.748381:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203430.748385:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203430.748388:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543302144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203430.748410:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203430.748413:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203430.748416:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203430.748417:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203430.748419:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203430.748420:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203430.748422:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25117 00000400:00000200:2.0:1788203430.748425:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203430.748427:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203430.748428:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203430.748430:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748432:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.748434:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748435:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203430.748437:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203430.748439:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203430.748440:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203430.748442:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203430.748443:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203430.748445:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203430.748447:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748448:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.748450:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748452:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203430.748455:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203430.748457:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748459:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203430.748460:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203430.748467:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203430.748469:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.748472:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203430.748476:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203430.748481:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.748484:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203430.748489:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788203430.748497:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203430.748500:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203430.748501:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.748502:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203430.748503:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203430.748505:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203430.748509:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203430.748514:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203430.748523:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543302272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203430.748585:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203430.748590:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203430.748595:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543302272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203430.748618:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203430.748622:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203430.748627:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203430.748635:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203430.748640:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203430.748646:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203430.748655:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25117 00000400:00000200:2.0:1788203430.748664:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203430.748669:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203430.748674:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203430.748683:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748686:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.748688:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748692:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203430.748699:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203430.748702:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203430.748704:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203430.748706:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203430.748707:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203430.748709:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203430.748712:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748714:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.748724:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748730:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203430.748735:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203430.748737:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748741:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203430.748743:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203430.748748:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203430.748753:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.748758:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203430.748761:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203430.748764:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.748766:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203430.748774:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788203430.748779:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203430.748782:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203430.748783:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.748785:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203430.748786:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203430.748788:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203430.748797:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203430.748806:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203430.748819:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543302400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203430.748844:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203430.748851:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203430.748866:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543302400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203430.748889:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203430.748892:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203430.748896:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203430.748900:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203430.748901:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203430.748904:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203430.748910:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25117 00000400:00000200:2.0:1788203430.748917:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203430.748921:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203430.748929:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203430.748940:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748947:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.748955:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.748963:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203430.748970:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203430.748974:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203430.748981:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203430.748985:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203430.749005:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203430.749009:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203430.749015:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.749023:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.749030:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.749034:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203430.749040:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203430.749050:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203430.749056:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203430.749060:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203430.749067:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203430.749073:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.749075:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203430.749078:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203430.749080:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.749086:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203430.749115:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203430.749118:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203430.749121:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203430.749123:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.749124:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203430.749125:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203430.749127:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203430.749130:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203430.749134:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203430.749139:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543302528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203430.749151:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203430.749157:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203430.749164:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543302528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203430.749180:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203430.749184:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.1:1788203430.749184:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788203430.749197:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203430.749199:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:0.1:1788203430.749199:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00080000:2.0:1788203430.749205:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203430.749215:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203430.749222:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25117 00000400:00000200:2.0:1788203430.749231:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203430.749238:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203430.749247:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203430.749253:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.749258:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.749265:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.749275:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203430.749282:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203430.749286:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203430.749295:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203430.749302:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203430.749308:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203430.749312:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203430.749317:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.749318:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203430.749320:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203430.749322:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203430.749325:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203430.749332:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203430.749340:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203430.749347:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203430.749353:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203430.749359:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.749363:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203430.749366:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203430.749372:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.749374:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203430.749381:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203430.749388:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203430.749396:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203430.749399:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203430.749410:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203430.749415:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203430.749421:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203430.749431:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203430.749436:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203430.749446:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543302656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203430.749452:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203430.749455:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203430.749462:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543302656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203430.749484:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203430.749496:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25127) 00000800:00000200:0.1:1788203430.749874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203430.749877:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203430.750231:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203430.750234:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203430.750559:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203430.750562:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203430.750865:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203430.750868:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788203431.580195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203431.580208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203431.580212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203431.580222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203431.580232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203431.580240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203431.580247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203431.580254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203431.580271:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203432.092218:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319790854, at 4319790912 00000100:00000010:2.0:1788203432.092242:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faf00. 00000100:00000001:2.0:1788203432.604150:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203432.604157:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203432.604161:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203432.604167:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203432.604175:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203432.604180:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203432.604187:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203432.604190:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203432.604194:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203433.628265:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203433.628277:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203433.628281:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203433.628286:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203433.628299:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203433.628305:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203433.628320:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203433.628335:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203433.628354:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203434.652192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203434.652198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203434.652202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203434.652213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203434.652218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203434.652220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203434.652226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203434.652232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203434.652239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203435.676198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203435.676207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203435.676211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203435.676220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203435.676237:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203435.676243:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203435.676246:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203435.676249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203435.676254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203435.868871:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203435.868878:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203435.869602:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203435.869606:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203435.869923:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203435.869925:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203435.870239:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203435.870242:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203435.870531:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203435.870534:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1788203436.700154:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203436.700163:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203436.700166:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203436.700175:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203436.700181:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203436.700185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203436.700190:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203436.700195:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203436.700207:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203437.212171:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319795973, at 4319796032 00000100:00000010:2.0:1788203437.212186:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa300. 00000100:00000001:2.0:1788203437.724191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203437.724200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203437.724203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203437.724211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203437.724217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203437.724222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203437.724228:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203437.724233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203437.724240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203438.748275:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203438.748281:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203438.748284:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203438.748287:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203438.748290:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203438.748292:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203438.748295:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203438.748297:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203438.748300:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203439.772171:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203439.772179:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203439.772182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203439.772188:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203439.772194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203439.772200:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203439.772205:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203439.772210:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203439.772216:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203440.796178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203440.796185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203440.796187:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203440.796192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203440.796197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203440.796201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203440.796204:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203440.796207:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203440.796210:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203440.988929:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203440.989323:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203440.989512:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203440.989661:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203440.989845:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203441.820179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203441.820187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203441.820190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203441.820197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203441.820203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203441.820208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203441.820211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203441.820215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203441.820223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203442.332205:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319801093, at 4319801152 00000100:00000010:2.0:1788203442.332220:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faba0. 00000100:00000001:2.0:1788203442.844204:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203442.844213:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203442.844217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203442.844225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203442.844233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203442.844240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203442.844247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203442.844252:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203442.844257:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203443.868194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203443.868202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203443.868207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203443.868215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203443.868223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203443.868230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203443.868236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203443.868241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203443.868249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788203444.060347:0:256123:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788203444.060391:0:256123:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788203444.060399:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203444.060403:0:256123:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788203444.060409:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203444.060414:0:256123:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788203444.060420:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203444.060424:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a100. 00000400:00000200:2.0:1788203444.060430:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203444.060436:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203444.060444:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b660 00000400:00000010:2.0:1788203444.060447:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b660. 00000100:00000001:2.0:1788203444.060454:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203444.060463:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39f80 x1875055543303424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060490:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203444.060495:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543303424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060503:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203444.060505:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203444.060519:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203444.060522:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a700. 00000400:00000200:2.0:1788203444.060525:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203444.060529:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203444.060533:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b6e8 00000400:00000010:2.0:1788203444.060535:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b6e8. 00000100:00000001:2.0:1788203444.060537:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203444.060542:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39c00 x1875055543303552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060551:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203444.060555:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543303552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060562:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203444.060564:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203444.060569:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203444.060571:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a200. 00000400:00000200:2.0:1788203444.060573:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203444.060577:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203444.060597:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b000 00000400:00000010:2.0:1788203444.060599:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b000. 00000100:00000001:2.0:1788203444.060601:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203444.060607:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a680 x1875055543303680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060615:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203444.060618:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543303680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060626:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203444.060628:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203444.060632:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203444.060635:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a600. 00000400:00000200:2.0:1788203444.060638:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203444.060642:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203444.060644:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b550 00000400:00000010:2.0:1788203444.060646:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b550. 00000100:00000001:2.0:1788203444.060649:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203444.060653:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3b480 x1875055543303808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060662:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203444.060665:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543303808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060671:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203444.060674:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203444.060679:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203444.060681:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868ab00. 00000400:00000200:2.0:1788203444.060684:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203444.060688:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203444.060691:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b2a8 00000400:00000010:2.0:1788203444.060693:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b2a8. 00000100:00000001:2.0:1788203444.060695:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203444.060700:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38000 x1875055543303936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060708:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203444.060711:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543303936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203465 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203444.060717:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203444.060719:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788203444.892189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203444.892199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203444.892203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203444.892209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203444.892216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203444.892221:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203444.892230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203444.892235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203444.892240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203445.916209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203445.916220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203445.916225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203445.916234:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203445.916246:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203445.916252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203445.916267:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203445.916272:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203445.916278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203446.108249:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203446.108264:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203446.108268:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203446.108274:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203446.108279:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203446.108289:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25132 00000400:00000200:2.0:1788203446.108301:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203446.108309:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203446.108314:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203446.108317:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108322:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.108324:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108327:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203446.108333:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203446.108336:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203446.108339:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203446.108344:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203446.108347:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203446.108350:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203446.108353:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108355:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.108357:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108359:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203446.108363:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203446.108367:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108369:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203446.108373:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203446.108377:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203446.108381:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.108384:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203446.108387:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203446.108395:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203446.108399:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203446.108405:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203446.108407:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.108413:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203446.108416:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203446.108421:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203446.108431:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203446.108439:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203446.108451:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543304064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203446.108463:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203446.108470:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203446.108475:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543304064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203446.108514:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203446.108518:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203446.108523:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203446.108525:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203446.108526:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203446.108530:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203446.108533:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25132 00000400:00000200:2.0:1788203446.108537:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203446.108540:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203446.108543:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203446.108546:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108548:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.108551:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108553:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203446.108556:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203446.108557:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203446.108559:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203446.108562:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203446.108567:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203446.108570:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203446.108573:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108575:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.108577:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108580:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203446.108583:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203446.108587:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108591:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203446.108593:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203446.108603:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203446.108607:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.108610:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203446.108617:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203446.108621:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.108623:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203446.108634:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203446.108637:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203446.108643:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203446.108646:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.108650:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203446.108656:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203446.108659:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203446.108666:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203446.108676:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203446.108690:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543304192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203446.108706:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203446.108710:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203446.108745:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543304192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203446.108783:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203446.108799:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203446.108808:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203446.108813:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203446.108818:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203446.108826:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203446.108834:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25132 00000400:00000200:2.0:1788203446.108853:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203446.108858:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203446.108861:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203446.108866:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108875:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.108886:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108890:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203446.108897:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203446.108902:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203446.108907:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203446.108916:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203446.108921:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203446.108928:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203446.108939:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108944:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.108950:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108956:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203446.108963:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203446.108970:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203446.108976:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203446.108984:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203446.108989:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203446.109003:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109009:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203446.109016:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203446.109026:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109032:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203446.109042:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203446.109048:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203446.109052:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203446.109058:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109068:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203446.109073:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203446.109076:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203446.109088:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203446.109136:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203446.109145:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543304320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203446.109156:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203446.109159:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203446.109164:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543304320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203446.109203:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203446.109214:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203446.109227:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203446.109234:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203446.109238:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203446.109251:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203446.109263:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25132 00000400:00000200:2.0:1788203446.109277:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203446.109290:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203446.109295:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203446.109313:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109321:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.109331:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109337:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203446.109346:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203446.109351:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203446.109356:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203446.109361:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203446.109372:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203446.109382:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203446.109396:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109405:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.109413:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109426:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:0.1:1788203446.109427:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:2.0:1788203446.109439:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203446.109447:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000200:0.1:1788203446.109447:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788203446.109457:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203446.109463:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203446.109471:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203446.109477:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109484:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203446.109487:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203446.109492:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109497:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203446.109508:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203446.109515:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203446.109531:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203446.109536:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109544:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203446.109547:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203446.109552:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203446.109562:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203446.109575:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203446.109589:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543304448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203446.109607:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203446.109615:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203446.109635:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543304448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203446.109659:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203446.109672:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203446.109686:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203446.109691:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203446.109694:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203446.109701:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203446.109707:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25132 00000400:00000200:2.0:1788203446.109717:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203446.109737:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203446.109743:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203446.109748:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109752:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.109759:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109768:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203446.109780:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203446.109785:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203446.109790:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203446.109796:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203446.109805:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203446.109812:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203446.109824:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109835:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203446.109849:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109858:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203446.109865:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203446.109873:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203446.109880:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203446.109889:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203446.109902:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203446.109914:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109924:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203446.109934:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203446.109946:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109958:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203446.109969:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203446.109977:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203446.109985:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203446.109987:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203446.109996:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203446.110000:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203446.110009:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203446.110023:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203446.110034:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203446.110049:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543304576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203446.110078:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203446.110084:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203446.110139:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543304576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203446.110173:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203446.110186:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25142) 00000800:00000200:0.1:1788203446.110463:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203446.110467:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203446.110786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203446.110789:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203446.111076:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203446.111078:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203446.111393:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203446.111395:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788203446.940207:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203446.940213:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203446.940216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203446.940223:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203446.940229:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203446.940234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203446.940242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203446.940246:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203446.940252:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203447.452176:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319806214, at 4319806272 00000100:00000010:2.0:1788203447.452184:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa4e0. 00000100:00000001:2.0:1788203447.964192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203447.964202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203447.964207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203447.964214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203447.964220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203447.964226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203447.964235:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203447.964240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203447.964244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203448.988199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203448.988207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203448.988212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203448.988218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203448.988223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203448.988228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203448.988233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203448.988237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203448.988244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203450.012169:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203450.012177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203450.012182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203450.012190:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203450.012197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203450.012204:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203450.012215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203450.012222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203450.012232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.036190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203451.036198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203451.036202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203451.036210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203451.036218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203451.036225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203451.036237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203451.036242:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203451.036247:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203451.228218:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203451.228227:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203451.228229:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203451.228231:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203451.228235:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203451.228241:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25137 00000400:00000200:2.0:1788203451.228250:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203451.228259:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203451.228262:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203451.228264:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228267:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.228269:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228272:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203451.228275:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203451.228277:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203451.228280:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203451.228283:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203451.228285:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203451.228287:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203451.228290:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228291:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.228293:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228295:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203451.228297:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203451.228300:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228303:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203451.228306:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203451.228309:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203451.228311:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.228313:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203451.228315:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203451.228324:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788203451.228327:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203451.228329:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203451.228331:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.228333:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203451.228335:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203451.228338:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203451.228345:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203451.228350:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203451.228357:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543304704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203451.228365:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203451.228369:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203451.228372:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543304704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203451.228388:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203451.228390:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203451.228392:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203451.228393:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203451.228395:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203451.228397:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203451.228399:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25137 00000400:00000200:2.0:1788203451.228403:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203451.228405:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203451.228407:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203451.228410:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228412:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.228415:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228422:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203451.228426:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203451.228430:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203451.228439:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203451.228443:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203451.228446:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203451.228451:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203451.228457:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228462:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.228463:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228464:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203451.228466:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203451.228478:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228488:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203451.228493:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203451.228507:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203451.228511:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.228514:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203451.228523:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203451.228531:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.228539:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203451.228544:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788203451.228547:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203451.228549:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203451.228554:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.228568:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203451.228573:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203451.228575:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203451.228579:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203451.228583:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203451.228600:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543304832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203451.228611:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203451.228618:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203451.228625:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543304832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203451.228648:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203451.228651:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203451.228654:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203451.228655:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203451.228657:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203451.228667:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203451.228677:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25137 00000400:00000200:2.0:1788203451.228683:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203451.228694:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203451.228704:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203451.228719:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228728:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.228743:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228751:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203451.228761:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203451.228762:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203451.228767:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203451.228771:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203451.228775:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203451.228786:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203451.228804:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228808:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.228816:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228824:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203451.228828:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203451.228836:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203451.228839:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203451.228840:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203451.228843:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203451.228845:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.228848:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203451.228853:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203451.228862:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.228870:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203451.228876:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788203451.228879:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203451.228884:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203451.228885:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.228887:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203451.228888:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203451.228889:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203451.228894:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203451.228897:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203451.228908:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543304960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203451.228932:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203451.228938:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203451.228949:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543304960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203451.228965:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203451.228972:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203451.228974:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203451.228977:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203451.228978:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203451.228982:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203451.228991:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25137 00000400:00000200:2.0:1788203451.229006:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203451.229014:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203451.229019:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:0.1:1788203451.229028:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788203451.229029:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229041:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1788203451.229042:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000040:2.0:1788203451.229048:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229058:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203451.229068:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203451.229071:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203451.229074:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203451.229082:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203451.229121:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203451.229125:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203451.229128:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229129:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.229131:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229132:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203451.229134:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203451.229136:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229138:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203451.229139:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203451.229143:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203451.229144:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.229146:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203451.229149:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203451.229151:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.229153:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203451.229157:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203451.229160:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203451.229163:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203451.229167:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.229169:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203451.229170:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203451.229172:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203451.229179:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203451.229184:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203451.229193:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543305088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203451.229209:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203451.229216:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203451.229226:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543305088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203451.229268:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203451.229271:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203451.229280:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203451.229289:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203451.229294:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203451.229301:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203451.229311:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25137 00000400:00000200:2.0:1788203451.229326:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203451.229332:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203451.229336:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203451.229344:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229347:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.229351:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229354:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203451.229356:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203451.229357:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203451.229359:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203451.229360:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203451.229363:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203451.229364:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203451.229367:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229368:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203451.229370:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229371:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203451.229372:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203451.229374:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203451.229376:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203451.229378:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203451.229380:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203451.229382:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.229383:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203451.229384:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203451.229385:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.229386:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203451.229388:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203451.229390:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203451.229392:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203451.229393:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203451.229395:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203451.229397:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203451.229399:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203451.229402:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203451.229405:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203451.229411:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543305216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203451.229425:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203451.229433:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203451.229442:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543305216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203451.229469:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203451.229481:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25147) 00000800:00000200:0.1:1788203451.229754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203451.229757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203451.229955:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203451.229957:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203451.230214:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203451.230215:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203451.230406:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203451.230407:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788203452.060220:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203452.060231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203452.060236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203452.060242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203452.060247:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203452.060252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203452.060256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203452.060259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203452.060263:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203452.572196:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319811334, at 4319811392 00000100:00000010:2.0:1788203452.572211:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fac60. 00000100:00000001:2.0:1788203453.084243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203453.084252:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203453.084263:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203453.084274:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203453.084286:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203453.084297:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203453.084311:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203453.084317:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203453.084329:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203454.108171:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203454.108178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203454.108182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203454.108187:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203454.108194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203454.108202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203454.108208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203454.108213:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203454.108216:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203455.132193:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203455.132202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203455.132206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203455.132213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203455.132218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203455.132223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203455.132229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203455.132233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203455.132239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203456.156164:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203456.156172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203456.156176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203456.156181:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203456.156188:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203456.156193:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203456.156198:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203456.156202:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203456.156209:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203456.348848:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203456.348857:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203456.349753:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203456.349759:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203456.350113:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203456.350116:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203456.350414:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203456.350417:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203456.350839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203456.350842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788203457.180190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203457.180199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203457.180204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203457.180209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203457.180216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203457.180222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203457.180229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203457.180234:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203457.180240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203457.692201:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319816453, at 4319816512 00000100:00000010:2.0:1788203457.692218:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fce40. 00000100:00000001:2.0:1788203458.204165:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203458.204173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203458.204178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203458.204186:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203458.204194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203458.204201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203458.204209:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203458.204214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203458.204223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203459.228181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203459.228189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203459.228192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203459.228199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203459.228207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203459.228214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203459.228221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203459.228225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203459.228231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203460.252194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203460.252203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203460.252207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203460.252214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203460.252218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203460.252223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203460.252228:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203460.252233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203460.252241:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203461.276199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203461.276207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203461.276212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203461.276218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203461.276224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203461.276230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203461.276236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203461.276240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203461.276242:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203461.468799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203461.468806:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203461.469638:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203461.469643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203461.469977:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203461.469980:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203461.470412:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203461.470415:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203461.470734:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203461.470736:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1788203462.300191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203462.300199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203462.300202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203462.300209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203462.300216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203462.300221:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203462.300227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203462.300230:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203462.300234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203462.812180:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319821573, at 4319821632 00000100:00000010:2.0:1788203462.812199:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc360. 00000100:00000001:2.0:1788203463.324184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203463.324191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203463.324196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203463.324204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203463.324211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203463.324218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203463.324225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203463.324231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203463.324238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203464.348208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203464.348218:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203464.348222:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203464.348229:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203464.348236:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203464.348241:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203464.348245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203464.348249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203464.348254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203465.372151:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203465.372159:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203465.372161:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203465.372167:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203465.372172:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203465.372176:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203465.372188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203465.372192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203465.372198:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203466.396189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203466.396197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203466.396200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203466.396206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203466.396215:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203466.396222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203466.396228:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203466.396233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203466.396237:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203466.588748:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203466.588753:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203466.589472:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203466.589475:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203466.589682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203466.589683:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203466.589861:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203466.589862:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203466.590121:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203466.590123:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1788203467.420181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203467.420188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203467.420192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203467.420197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203467.420202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203467.420208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203467.420215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203467.420219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203467.420224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203467.932202:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319826693, at 4319826752 00000100:00000010:2.0:1788203467.932222:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcc60. 00000100:00000001:2.0:1788203468.444196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203468.444204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203468.444207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203468.444213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203468.444222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203468.444228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203468.444234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203468.444240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203468.444247:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203469.468203:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203469.468209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203469.468213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203469.468218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203469.468223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203469.468228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203469.468232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203469.468236:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203469.468240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203470.492220:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203470.492231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203470.492235:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203470.492243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203470.492253:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203470.492263:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203470.492271:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203470.492278:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203470.492286:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203471.516226:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203471.516239:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203471.516245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203471.516257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203471.516269:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203471.516279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203471.516300:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203471.516303:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203471.516307:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203471.708969:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203471.708977:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203471.709831:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203471.709834:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203471.710075:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203471.710079:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203471.710407:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203471.710411:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203471.710733:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203471.710735:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1788203472.540204:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203472.540216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203472.540219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203472.540227:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203472.540231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203472.540235:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203472.540242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203472.540247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203472.540251:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203473.052224:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319831814, at 4319831872 00000100:00000010:2.0:1788203473.052249:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa6c0. 00000100:00000001:2.0:1788203473.564178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203473.564185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203473.564187:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203473.564192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203473.564198:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203473.564204:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203473.564211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203473.564214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203473.564229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203474.588192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203474.588203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203474.588210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203474.588218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203474.588239:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203474.588245:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203474.588252:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203474.588256:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203474.588264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203475.612182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203475.612190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203475.612194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203475.612201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203475.612207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203475.612223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203475.612226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203475.612229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203475.612232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203476.636227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203476.636281:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203476.636284:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203476.636287:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203476.636292:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203476.636295:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203476.636299:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203476.636301:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203476.636303:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203476.828733:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203476.828742:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203476.829941:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203476.829946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203476.830547:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203476.830550:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203476.830777:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203476.830792:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203476.830976:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203476.830978:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788203477.660196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203477.660218:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203477.660221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203477.660226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203477.660232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203477.660238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203477.660242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203477.660247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203477.660254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203478.173464:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319836933, at 4319836993 00000100:00000010:2.0:1788203478.173471:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa420. 00000100:00000001:2.0:1788203478.684207:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203478.684216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203478.684221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203478.684227:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203478.684236:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203478.684241:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203478.684249:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203478.684254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203478.684259:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203479.708208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203479.708217:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203479.708220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203479.708226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203479.708231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203479.708238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203479.708249:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203479.708254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203479.708258:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203480.732222:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203480.732236:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203480.732240:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203480.732246:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203480.732253:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203480.732260:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203480.732266:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203480.732271:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203480.732274:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203481.756201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203481.756208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203481.756213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203481.756220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203481.756226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203481.756230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203481.756234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203481.756240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203481.756244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203481.948842:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203481.948863:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203481.949532:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203481.949535:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203481.949791:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203481.949794:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203481.949976:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203481.949978:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203481.950224:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203481.950227:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788203482.780201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203482.780211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203482.780215:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203482.780225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203482.780235:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203482.780240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203482.780247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203482.780250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203482.780255:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203483.292256:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319842053, at 4319842112 00000100:00000010:2.0:1788203483.292273:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc120. 00000100:00000001:2.0:1788203483.804226:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203483.804236:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203483.804240:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203483.804251:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203483.804258:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203483.804266:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203483.804274:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203483.804279:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203483.804286:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203484.828199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203484.828209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203484.828213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203484.828221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203484.828228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203484.828234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203484.828241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203484.828247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203484.828252:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203485.852196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203485.852204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203485.852209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203485.852217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203485.852224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203485.852230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203485.852238:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203485.852243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203485.852251:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203486.876169:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203486.876182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203486.876186:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203486.876192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203486.876199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203486.876204:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203486.876211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203486.876217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203486.876221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203487.068671:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203487.068675:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203487.069378:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203487.069383:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203487.069711:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203487.069713:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203487.069987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203487.069989:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203487.070220:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203487.070222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788203487.900191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203487.900200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203487.900204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203487.900211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203487.900219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203487.900224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203487.900229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203487.900233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203487.900238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203488.412171:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319847173, at 4319847232 00000100:00000010:2.0:1788203488.412187:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc540. 00000100:00000001:2.0:1788203488.924162:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203488.924169:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203488.924172:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203488.924180:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203488.924186:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203488.924191:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203488.924197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203488.924201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203488.924206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203489.948210:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203489.948218:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203489.948222:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203489.948230:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203489.948238:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203489.948244:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203489.948250:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203489.948254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203489.948258:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203490.972167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203490.972177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203490.972182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203490.972190:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203490.972199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203490.972206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203490.972214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203490.972219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203490.972224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203491.996198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203491.996208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203491.996213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203491.996222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203491.996230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203491.996238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203491.996247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203491.996253:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203491.996259:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203492.188729:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203492.188736:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203492.189452:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203492.189454:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203492.189650:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203492.189652:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203492.189835:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203492.189837:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203492.190187:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203492.190189:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788203493.020196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203493.020211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203493.020214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203493.020218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203493.020224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203493.020230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203493.020236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203493.020240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203493.020244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203493.532197:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319852293, at 4319852352 00000100:00000010:2.0:1788203493.532221:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc900. 00000100:00000001:2.0:1788203494.044167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203494.044175:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203494.044176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203494.044183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203494.044190:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203494.044195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203494.044202:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203494.044206:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203494.044210:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203495.068196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203495.068203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203495.068206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203495.068212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203495.068217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203495.068221:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203495.068224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203495.068228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203495.068236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203496.092214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203496.092224:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203496.092229:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203496.092238:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203496.092246:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203496.092253:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203496.092261:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203496.092265:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203496.092269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203497.116187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203497.116196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203497.116202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203497.116210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203497.116217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203497.116224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203497.116230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203497.116235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203497.116241:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203497.308863:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203497.308869:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203497.309687:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203497.309689:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203497.309935:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203497.309937:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203497.310221:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203497.310224:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203497.310522:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203497.310524:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788203498.140232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203498.140240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203498.140246:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203498.140253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203498.140260:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203498.140263:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203498.140267:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203498.140272:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203498.140277:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203498.652193:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319857413, at 4319857472 00000100:00000010:2.0:1788203498.652210:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc240. 00000100:00000001:2.0:1788203499.164237:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203499.164249:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203499.164260:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203499.164271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203499.164282:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203499.164291:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203499.164302:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203499.164308:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203499.164313:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203500.188249:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203500.188260:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203500.188270:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203500.188286:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203500.188296:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203500.188308:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203500.188316:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203500.188330:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203500.188355:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203500.188404:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203500.188408:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203500.188412:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788203501.212180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203501.212189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203501.212227:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203501.212234:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203501.212250:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203501.212257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203501.212265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203501.212271:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203501.212275:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203502.236171:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203502.236180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203502.236185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203502.236191:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203502.236199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203502.236204:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203502.236211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203502.236216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203502.236222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203502.428775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203502.428782:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203502.429442:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203502.429443:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203502.429629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203502.429630:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203502.429851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203502.429853:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203502.430122:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203502.430123:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1788203503.260184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203503.260192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203503.260194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203503.260200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203503.260205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203503.260209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203503.260212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203503.260218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203503.260224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203503.260371:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203503.260373:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203503.260375:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203503.772152:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319862533, at 4319862592 00000100:00000010:2.0:1788203503.772169:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc7e0. 00000100:00000001:2.0:1788203504.284192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203504.284202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203504.284206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203504.284211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203504.284217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203504.284223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203504.284227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203504.284232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203504.284248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203505.308172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203505.308180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203505.308185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203505.308192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203505.308195:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203505.308200:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203505.308206:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203505.308211:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203505.308218:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203506.332167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203506.332175:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203506.332178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203506.332184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203506.332189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203506.332194:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203506.332199:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203506.332203:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203506.332210:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203506.332271:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203506.332274:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203506.332276:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788203507.356208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203507.356220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203507.356225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203507.356237:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203507.356253:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203507.356263:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203507.356270:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203507.356275:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203507.356283:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203507.548850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203507.549219:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203507.549626:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203507.549916:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203507.550134:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203508.380162:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203508.380168:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203508.380171:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203508.380176:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203508.380183:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203508.380188:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203508.380192:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203508.380196:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203508.380202:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203508.892198:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319867653, at 4319867712 00000100:00000010:2.0:1788203508.892226:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fade0. 00000100:00000001:2.0:1788203509.404209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203509.404220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203509.404224:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203509.404232:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203509.404240:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203509.404247:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203509.404254:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203509.404259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203509.404265:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203510.428187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203510.428195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203510.428198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203510.428207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203510.428215:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203510.428220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203510.428226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203510.428231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203510.428236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203511.452231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203511.452243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203511.452247:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203511.452255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203511.452261:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203511.452266:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203511.452272:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203511.452278:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203511.452286:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203512.476239:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203512.476252:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203512.476258:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203512.476269:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203512.476278:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203512.476284:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203512.476288:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203512.476292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203512.476298:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203512.669032:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203512.669038:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203512.674455:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203512.674460:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203512.674850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203512.674852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203512.675228:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203512.675231:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203512.675708:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203512.675712:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788203513.500178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203513.500189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203513.500193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203513.500200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203513.500206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203513.500211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203513.500217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203513.500221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203513.500227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203514.012207:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319872774, at 4319872832 00000100:00000010:2.0:1788203514.012224:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faae0. 00000100:00000001:2.0:1788203514.524184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203514.524193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203514.524196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203514.524203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203514.524210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203514.524215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203514.524221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203514.524226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203514.524239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203515.548198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203515.548208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203515.548211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203515.548215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203515.548219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203515.548223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203515.548229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203515.548232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203515.548235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203516.572336:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203516.572340:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203516.572342:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203516.572345:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203516.572347:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203516.572349:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203516.572352:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203516.572354:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203516.572356:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203517.596197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203517.596204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203517.596207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203517.596213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203517.596218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203517.596222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203517.596226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203517.596229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203517.596235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203517.788912:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203517.788919:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203517.789833:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203517.789838:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203517.790182:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203517.790185:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203517.790506:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203517.790509:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203517.790933:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203517.790936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788203518.620178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203518.620185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203518.620189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203518.620196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203518.620202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203518.620208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203518.620215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203518.620219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203518.620222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203519.132209:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319877893, at 4319877952 00000100:00000010:2.0:1788203519.132226:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faa20. 00000100:00000001:2.0:1788203519.644198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203519.644211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203519.644216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203519.644232:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203519.644245:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203519.644249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203519.644256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203519.644260:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203519.644263:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203520.668207:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203520.668220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203520.668227:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203520.668235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203520.668255:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203520.668271:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203520.668278:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203520.668282:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203520.668292:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203521.692236:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203521.692247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203521.692252:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203521.692262:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203521.692270:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203521.692276:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203521.692280:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203521.692290:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203521.692297:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203522.716231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203522.716240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203522.716245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203522.716254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203522.716259:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203522.716265:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203522.716273:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203522.716277:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203522.716282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203522.909077:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203522.909122:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203522.909558:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203522.909560:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203522.909886:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203522.909889:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203522.910161:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203522.910163:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:0.1:1788203522.910426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203522.910428:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788203523.740179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203523.740188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203523.740192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203523.740198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203523.740203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203523.740209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203523.740216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203523.740222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203523.740226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203524.252186:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319883014, at 4319883072 00000100:00000010:2.0:1788203524.252202:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa720. 00000100:00000001:2.0:1788203524.764184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203524.764192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203524.764196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203524.764202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203524.764209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203524.764215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203524.764220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203524.764225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203524.764228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203525.788212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203525.788230:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203525.788235:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203525.788246:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203525.788257:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203525.788264:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203525.788270:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203525.788274:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203525.788280:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203526.812179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203526.812186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203526.812189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203526.812195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203526.812201:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203526.812204:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203526.812208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203526.812212:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203526.812217:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203527.836154:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203527.836162:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203527.836167:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203527.836173:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203527.836180:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203527.836185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203527.836189:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203527.836192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203527.836197:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203528.028669:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203528.028673:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203528.029181:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203528.029183:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203528.029404:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203528.029406:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203528.029639:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203528.029640:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:0.1:1788203528.029803:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203528.029804:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1788203528.860237:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203528.860245:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203528.860248:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203528.860254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203528.860259:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203528.860263:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203528.860267:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203528.860270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203528.860274:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203529.372250:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319888133, at 4319888192 00000100:00000010:2.0:1788203529.372271:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa2a0. 00000100:00000001:2.0:1788203529.884202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203529.884211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203529.884214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203529.884224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203529.884240:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203529.884252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203529.884258:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203529.884260:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203529.884263:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203530.908247:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203530.908284:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203530.908290:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203530.908298:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203530.908307:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203530.908314:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203530.908325:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203530.908333:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203530.908339:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203531.932218:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203531.932230:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203531.932247:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203531.932252:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203531.932257:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203531.932261:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203531.932264:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203531.932267:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203531.932270:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203532.956206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203532.956227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203532.956234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203532.956245:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203532.956251:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203532.956259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203532.956263:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203532.956270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203532.956274:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203533.148995:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203533.149005:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203533.149782:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203533.149785:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203533.150019:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203533.150021:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203533.150278:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203533.150280:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203533.150556:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203533.150559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1788203533.980181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203533.980191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203533.980194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203533.980210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203533.980218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203533.980224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203533.980229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203533.980234:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203533.980240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203534.492252:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319893254, at 4319893312 00000100:00000010:2.0:1788203534.492280:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa780. 00000100:00000001:2.0:1788203535.004224:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203535.004234:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203535.004238:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203535.004245:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203535.004251:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203535.004258:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203535.004266:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203535.004272:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203535.004279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203536.028236:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203536.028246:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203536.028251:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203536.028259:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203536.028267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203536.028273:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203536.028281:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203536.028285:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203536.028292:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203537.052251:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203537.052261:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203537.052265:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203537.052273:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203537.052280:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203537.052286:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203537.052294:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203537.052300:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203537.052308:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203538.076250:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203538.076267:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203538.076271:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203538.076290:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203538.076312:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203538.076326:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203538.076343:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203538.076356:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203538.076372:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203538.269172:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203538.269181:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788203538.269870:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203538.269873:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788203538.270231:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203538.270234:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788203538.270526:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203538.270529:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788203538.270825:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203538.270828:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1788203539.100206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203539.100214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203539.100216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203539.100221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203539.100225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203539.100229:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203539.100236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203539.100239:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203539.100248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203539.612202:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319898374, at 4319898432 00000100:00000010:2.0:1788203539.612218:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcea0. 00000100:00000001:2.0:1788203540.124216:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203540.124224:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203540.124231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203540.124241:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203540.124251:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203540.124257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203540.124264:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203540.124269:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203540.124273:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203541.148265:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203541.148282:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203541.148293:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203541.148305:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203541.148325:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203541.148334:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203541.148344:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203541.148348:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203541.148354:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203542.172195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203542.172204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203542.172207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203542.172213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203542.172218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203542.172223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203542.172228:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203542.172231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203542.172236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203543.196186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203543.196195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203543.196200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203543.196210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203543.196218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203543.196225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203543.196231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203543.196237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203543.196244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203543.196280:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203543.196283:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203543.196287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788203543.388735:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203543.388741:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788203543.389499:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203543.389504:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788203543.389886:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203543.389888:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788203543.390197:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203543.390199:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788203543.390409:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203543.390411:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000100:00000001:2.0:1788203544.220154:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203544.220162:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203544.220165:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203544.220170:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203544.220175:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203544.220181:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203544.220188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203544.220193:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203544.220197:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203544.732154:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319903493, at 4319903552 00000100:00000010:2.0:1788203544.732168:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf00. 00000100:00000001:2.0:1788203545.244160:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203545.244169:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203545.244174:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203545.244180:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203545.244188:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203545.244194:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203545.244202:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203545.244206:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203545.244214:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203546.268189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203546.268197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203546.268202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203546.268207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203546.268211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203546.268215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203546.268222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203546.268225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203546.268230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203546.268262:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203546.268266:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203546.268270:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788203547.292174:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203547.292183:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203547.292188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203547.292197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203547.292203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203547.292208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203547.292217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203547.292221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203547.292229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203548.316194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203548.316204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203548.316209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203548.316216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203548.316224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203548.316230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203548.316234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203548.316240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203548.316248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203548.508737:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203548.508742:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203548.509322:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203548.509324:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203548.509534:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203548.509536:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203548.509760:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203548.509762:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203548.509940:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203548.509942:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788203549.340181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203549.340190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203549.340194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203549.340202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203549.340210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203549.340215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203549.340225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203549.340231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203549.340235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203549.852172:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319908613, at 4319908672 00000100:00000010:2.0:1788203549.852193:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc660. 00000100:00000001:2.0:1788203550.364204:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203550.364210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203550.364214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203550.364221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203550.364225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203550.364231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203550.364237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203550.364241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203550.364249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203551.388172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203551.388180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203551.388183:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203551.388188:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203551.388194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203551.388196:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203551.388203:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203551.388208:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203551.388211:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203552.412200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203552.412210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203552.412214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203552.412218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203552.412225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203552.412231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203552.412238:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203552.412243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203552.412248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203553.436171:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203553.436182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203553.436185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203553.436193:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203553.436200:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203553.436204:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203553.436209:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203553.436217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203553.436231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203553.628807:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203553.628811:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203553.629695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203553.629700:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203553.629986:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203553.629988:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203553.630320:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203553.630322:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203553.630536:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203553.630538:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788203554.460166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203554.460174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203554.460177:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203554.460184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203554.460193:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203554.460199:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203554.460208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203554.460214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203554.460221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203554.972200:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319913733, at 4319913792 00000100:00000010:2.0:1788203554.972216:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fccc0. 00000100:00000001:2.0:1788203555.484180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203555.484188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203555.484192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203555.484200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203555.484207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203555.484212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203555.484218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203555.484222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203555.484228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203556.508189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203556.508198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203556.508206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203556.508214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203556.508219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203556.508223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203556.508229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203556.508233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203556.508239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203557.532229:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203557.532240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203557.532245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203557.532255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203557.532271:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203557.532281:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203557.532297:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203557.532303:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203557.532311:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203558.556195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203558.556204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203558.556206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203558.556219:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203558.556232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203558.556243:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203558.556247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203558.556254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203558.556265:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203558.748947:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203558.748953:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203558.750144:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203558.750147:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203558.750533:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203558.750535:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203558.750795:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203558.750797:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203558.750980:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203558.750982:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788203559.580167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203559.580177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203559.580179:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203559.580185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203559.580190:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203559.580194:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203559.580197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203559.580201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203559.580205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203560.092217:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319918854, at 4319918912 00000100:00000010:2.0:1788203560.092236:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcd80. 00000100:00000001:2.0:1788203560.604194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203560.604202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203560.604205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203560.604210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203560.604215:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203560.604220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203560.604225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203560.604233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203560.604246:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203561.628235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203561.628250:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203561.628258:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203561.628265:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203561.628277:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203561.628283:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203561.628291:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203561.628293:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203561.628313:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203562.652178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203562.652185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203562.652189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203562.652196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203562.652200:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203562.652205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203562.652214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203562.652224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203562.652238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203563.676196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203563.676207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203563.676213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203563.676223:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203563.676232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203563.676239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203563.676248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203563.676253:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203563.676262:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203563.868892:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203563.868905:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203563.869629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203563.869630:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203563.869889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203563.869891:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203563.870128:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203563.870130:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203563.870491:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203563.870493:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788203564.700167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203564.700175:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203564.700178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203564.700186:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203564.700193:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203564.700196:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203564.700201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203564.700205:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203564.700212:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203565.212217:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319923973, at 4319924032 00000100:00000010:2.0:1788203565.212231:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fccc0. 00000100:00000001:2.0:1788203565.724206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203565.724217:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203565.724223:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203565.724231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203565.724240:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203565.724246:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203565.724256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203565.724259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203565.724264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203566.748183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203566.748196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203566.748203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203566.748213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203566.748223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203566.748231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203566.748238:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203566.748244:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203566.748251:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203567.772229:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203567.772241:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203567.772245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203567.772262:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203567.772268:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203567.772274:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203567.772281:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203567.772285:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203567.772289:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203568.988976:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203568.988990:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203568.989799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203568.989802:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203568.989999:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203568.990001:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203568.990210:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203568.990212:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203568.990429:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203568.990431:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000040:2.0:1788203570.332201:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319929094, at 4319929152 00000100:00000010:2.0:1788203570.332219:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc660. 00000100:00000001:2.0:1788203570.844243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203570.844251:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203570.844253:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203570.844257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203570.844262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203570.844265:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203570.844267:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203570.844270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203570.844273:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203571.868213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203571.868223:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203571.868228:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203571.868236:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203571.868250:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203571.868267:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203571.868289:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203571.868293:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203571.868307:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203572.892231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203572.892248:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203572.892255:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203572.892282:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203572.892290:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203572.892298:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203572.892308:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203572.892314:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203572.892329:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203573.916212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203573.916220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203573.916226:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203573.916240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203573.916250:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203573.916262:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203573.916272:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203573.916279:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203573.916287:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203574.108239:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203574.108255:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203574.108259:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203574.108264:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203574.108270:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203574.108279:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25260 00000400:00000200:2.0:1788203574.108293:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203574.108300:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203574.108304:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203574.108307:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108309:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.108312:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108313:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203574.108317:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203574.108319:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203574.108322:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203574.108325:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203574.108327:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203574.108329:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203574.108331:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108333:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.108335:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108336:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203574.108339:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203574.108343:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108345:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203574.108347:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203574.108350:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203574.108353:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108355:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203574.108357:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203574.108368:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203574.108372:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203574.108377:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203574.108379:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108383:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203574.108387:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203574.108392:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203574.108402:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203574.108412:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203574.108425:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543320064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203574.108438:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203574.108444:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203574.108449:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543320064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203574.108484:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203574.108488:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203574.108492:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203574.108493:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203574.108495:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203574.108528:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203574.108533:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25260 00000400:00000200:2.0:1788203574.108540:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203574.108546:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203574.108551:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203574.108557:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108562:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.108567:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108573:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203574.108579:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203574.108582:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203574.108586:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203574.108592:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203574.108597:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203574.108600:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203574.108604:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108607:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.108611:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108614:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203574.108620:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203574.108624:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108627:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203574.108629:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203574.108636:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203574.108639:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108642:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203574.108646:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203574.108650:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108654:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203574.108660:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203574.108662:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203574.108667:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203574.108671:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108673:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203574.108674:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203574.108677:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203574.108683:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203574.108688:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203574.108695:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543320192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203574.108703:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203574.108708:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203574.108714:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543320192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203574.108734:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203574.108739:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203574.108745:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203574.108749:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203574.108751:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203574.108754:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203574.108759:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25260 00000400:00000200:2.0:1788203574.108765:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203574.108767:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203574.108769:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203574.108775:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108778:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.108782:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108787:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203574.108790:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203574.108791:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203574.108793:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203574.108795:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203574.108799:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203574.108801:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203574.108806:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108809:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.108811:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108813:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203574.108815:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203574.108821:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108825:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203574.108829:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203574.108831:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203574.108834:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108837:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203574.108839:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203574.108845:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108848:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203574.108853:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203574.108857:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203574.108862:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203574.108865:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108868:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203574.108870:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203574.108872:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203574.108875:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203574.108880:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203574.108888:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543320320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203574.108895:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203574.108898:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203574.108900:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543320320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203574.108910:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203574.108914:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788203574.108918:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788203574.108919:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203574.108921:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203574.108924:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203574.108926:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203574.108929:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25260 00000400:00000200:2.0:1788203574.108931:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203574.108933:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203574.108935:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203574.108937:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108938:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.108940:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108942:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203574.108946:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203574.108947:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203574.108949:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203574.108951:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203574.108957:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203574.108959:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203574.108966:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108968:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.108969:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108972:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203574.108977:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203574.108979:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203574.108981:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203574.108982:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203574.108984:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203574.108989:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.108991:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203574.108994:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203574.109009:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.109013:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203574.109017:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203574.109023:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203574.109025:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203574.109027:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.109028:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203574.109029:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203574.109030:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203574.109034:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203574.109038:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203574.109043:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543320448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203574.109053:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203574.109057:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203574.109062:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543320448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203574.109139:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203574.109144:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203574.109150:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203574.109153:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203574.109156:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203574.109163:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203574.109167:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25260 00000400:00000200:2.0:1788203574.109173:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203574.109180:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203574.109183:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203574.109188:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.109190:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.109194:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.109201:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203574.109204:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203574.109206:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203574.109208:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203574.109210:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203574.109213:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203574.109215:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203574.109223:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.109227:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203574.109231:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203574.109237:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203574.109241:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203574.109246:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203574.109250:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203574.109257:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203574.109264:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203574.109269:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.109274:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203574.109279:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203574.109286:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.109290:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203574.109295:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203574.109300:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203574.109305:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203574.109307:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203574.109312:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203574.109314:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203574.109317:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203574.109324:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203574.109330:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203574.109339:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543320576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203574.109368:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203574.109373:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203574.109383:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788203574.109385:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543320576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203574.109401:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203574.109411:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25270) 00000800:00000200:3.1:1788203574.109590:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203574.109752:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203574.109923:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203574.940216:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203574.940226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203574.940229:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203574.940236:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203574.940249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203574.940254:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203574.940259:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203574.940263:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203574.940267:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203575.452186:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319934213, at 4319934272 00000100:00000010:2.0:1788203575.452214:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf00. 00000100:00000001:2.0:1788203575.964221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203575.964233:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203575.964236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203575.964247:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203575.964257:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203575.964261:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203575.964274:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203575.964278:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203575.964286:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203576.988256:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203576.988270:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203576.988282:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203576.988297:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203576.988306:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203576.988315:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203576.988322:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203576.988325:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203576.988338:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203578.012213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203578.012225:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203578.012227:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203578.012234:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203578.012239:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203578.012248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203578.012257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203578.012262:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203578.012267:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.036235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203579.036246:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203579.036251:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203579.036257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203579.036265:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203579.036271:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203579.036279:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203579.036286:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203579.036296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203579.228168:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203579.228180:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203579.228183:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203579.228187:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203579.228192:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203579.228200:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25265 00000400:00000200:2.0:1788203579.228211:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203579.228218:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203579.228221:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203579.228224:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228227:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.228229:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228231:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203579.228235:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203579.228237:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203579.228240:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203579.228245:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203579.228247:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203579.228249:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203579.228252:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228254:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.228255:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228256:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203579.228259:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203579.228262:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228264:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203579.228267:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203579.228270:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203579.228272:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.228274:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203579.228277:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203579.228284:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788203579.228287:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203579.228291:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203579.228293:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.228296:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203579.228299:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203579.228304:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203579.228311:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203579.228318:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203579.228328:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543320704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203579.228337:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203579.228341:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203579.228345:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543320704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203579.228375:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203579.228380:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203579.228385:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203579.228391:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203579.228393:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203579.228397:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203579.228400:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25265 00000400:00000200:2.0:1788203579.228404:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203579.228408:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203579.228424:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203579.228432:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228435:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.228439:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228442:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203579.228445:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203579.228448:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203579.228452:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203579.228456:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203579.228461:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203579.228465:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203579.228470:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228475:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.228478:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228483:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203579.228486:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203579.228491:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228494:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203579.228510:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203579.228521:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203579.228528:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.228532:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203579.228552:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203579.228557:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.228561:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203579.228571:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788203579.228576:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203579.228583:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203579.228586:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.228588:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203579.228591:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203579.228595:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203579.228605:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203579.228613:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203579.228621:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543320832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203579.228638:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203579.228644:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203579.228655:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543320832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203579.228674:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203579.228680:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203579.228687:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203579.228689:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203579.228693:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203579.228712:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203579.228718:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25265 00000400:00000200:2.0:1788203579.228785:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203579.228793:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203579.228809:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203579.228818:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228830:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1788203579.228830:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203579.228838:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000040:2.0:1788203579.228841:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228849:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203579.228855:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203579.228858:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203579.228862:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203579.228865:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203579.228872:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203579.228878:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203579.228886:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228889:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.228894:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228899:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203579.228905:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203579.228911:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203579.228917:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203579.228923:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203579.228928:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203579.228933:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.228936:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203579.228942:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203579.228947:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.228953:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203579.228960:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788203579.228965:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203579.228971:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203579.228974:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.228977:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203579.228979:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203579.228982:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203579.228989:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203579.228993:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203579.228999:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543320960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203579.229014:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203579.229016:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203579.229021:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543320960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203579.229029:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203579.229032:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203579.229035:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203579.229037:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203579.229039:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203579.229042:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203579.229046:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25265 00000400:00000200:2.0:1788203579.229050:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203579.229054:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203579.229057:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203579.229060:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229063:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.229064:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229066:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203579.229069:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203579.229070:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203579.229072:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203579.229074:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203579.229076:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203579.229079:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203579.229082:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229087:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.229126:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229132:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203579.229139:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203579.229143:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229149:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203579.229153:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203579.229157:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203579.229161:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.229166:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203579.229170:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203579.229175:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.229179:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203579.229188:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203579.229193:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203579.229199:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203579.229203:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.229206:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203579.229211:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203579.229214:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203579.229220:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203579.229228:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203579.229238:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543321088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203579.229253:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203579.229258:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203579.229266:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543321088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203579.229285:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203579.229292:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203579.229305:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203579.229309:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203579.229312:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203579.229320:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203579.229327:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25265 00000400:00000200:2.0:1788203579.229337:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203579.229343:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203579.229349:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203579.229355:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229360:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.229366:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229372:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203579.229378:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203579.229381:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203579.229385:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203579.229389:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203579.229394:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203579.229399:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203579.229408:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229434:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203579.229439:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229445:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203579.229450:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203579.229458:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203579.229465:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203579.229472:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203579.229477:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203579.229483:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.229488:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203579.229494:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203579.229501:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.229506:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203579.229514:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203579.229519:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203579.229525:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203579.229531:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203579.229534:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203579.229538:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203579.229543:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203579.229552:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203579.229559:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203579.229568:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543321216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203579.229582:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203579.229589:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203579.229597:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543321216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203579.229608:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203579.229616:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25275) 00000800:00000200:3.1:1788203579.229775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203579.229778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203579.230122:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203579.230125:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203579.230448:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203579.230451:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203579.230745:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203579.230748:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788203580.060230:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203580.060242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203580.060248:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203580.060257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203580.060270:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203580.060279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203580.060284:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203580.060290:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203580.060295:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203580.572220:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319939334, at 4319939392 00000100:00000010:2.0:1788203580.572243:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcea0. 00000100:00000001:2.0:1788203581.084233:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203581.084246:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203581.084254:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203581.084266:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203581.084276:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203581.084283:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203581.084288:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203581.084294:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203581.084302:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203582.108212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203582.108225:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203582.108230:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203582.108240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203582.108262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203582.108268:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203582.108273:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203582.108277:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203582.108281:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203583.132254:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203583.132266:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203583.132270:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203583.132281:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203583.132286:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203583.132296:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203583.132301:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203583.132305:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203583.132309:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.156233:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203584.156243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203584.156253:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203584.156273:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203584.156282:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203584.156288:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203584.156294:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203584.156300:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203584.156304:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203584.348227:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203584.348238:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203584.348241:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203584.348245:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203584.348250:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203584.348258:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25270 00000400:00000200:2.0:1788203584.348269:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203584.348277:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203584.348280:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203584.348282:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348286:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.348288:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348290:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203584.348294:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203584.348296:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203584.348298:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203584.348302:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203584.348303:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203584.348305:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203584.348307:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348309:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.348310:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348312:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203584.348314:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203584.348317:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348319:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203584.348346:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203584.348350:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203584.348352:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.348354:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203584.348357:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203584.348369:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203584.348374:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203584.348379:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203584.348381:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.348385:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203584.348389:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203584.348395:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203584.348406:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203584.348415:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203584.348427:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543321344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203584.348444:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203584.348449:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203584.348452:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543321344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203584.348475:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203584.348478:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203584.348481:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203584.348482:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203584.348483:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203584.348486:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203584.348488:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25270 00000400:00000200:2.0:1788203584.348492:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203584.348495:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203584.348496:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203584.348499:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348500:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.348502:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348503:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203584.348505:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203584.348506:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203584.348507:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203584.348509:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203584.348510:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203584.348511:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203584.348513:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348514:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.348516:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348517:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203584.348520:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203584.348522:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348524:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203584.348525:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203584.348532:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203584.348533:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.348535:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203584.348539:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203584.348542:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.348544:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203584.348548:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203584.348550:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203584.348553:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203584.348555:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.348556:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203584.348557:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203584.348559:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203584.348562:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203584.348566:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203584.348573:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543321472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203584.348589:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203584.348594:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203584.348604:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543321472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203584.348644:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203584.348651:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203584.348661:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203584.348671:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203584.348674:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203584.348680:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203584.348686:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25270 00000400:00000200:2.0:1788203584.348690:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203584.348692:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203584.348693:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203584.348701:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348703:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.348714:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348723:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203584.348727:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203584.348729:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203584.348732:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203584.348739:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203584.348746:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203584.348748:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203584.348751:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348753:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.348758:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348762:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203584.348769:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203584.348772:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203584.348774:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203584.348775:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203584.348783:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203584.348789:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.348792:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203584.348794:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203584.348795:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.348797:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203584.348809:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203584.348821:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203584.348829:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203584.348832:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.348838:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203584.348844:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203584.348849:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203584.348860:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203584.348872:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203584.348887:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543321600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203584.348906:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203584.348908:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203584.348916:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543321600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203584.348941:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203584.348952:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203584.348967:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203584.348974:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203584.348981:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203584.348991:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203584.348998:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25270 00000400:00000200:2.0:1788203584.349021:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203584.349029:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203584.349032:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203584.349044:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349048:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.349058:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349064:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788203584.349066:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788203584.349073:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:3.1:1788203584.349074:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000020:00000040:2.0:1788203584.349076:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203584.349086:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203584.349112:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203584.349115:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203584.349117:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203584.349122:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349124:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.349125:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349127:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203584.349128:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203584.349131:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349133:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203584.349134:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203584.349137:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203584.349139:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.349142:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203584.349145:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203584.349148:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.349150:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203584.349153:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203584.349154:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203584.349164:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203584.349169:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.349172:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203584.349175:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203584.349178:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203584.349182:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203584.349191:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203584.349195:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543321728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203584.349226:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203584.349236:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203584.349257:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543321728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203584.349315:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203584.349333:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203584.349337:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203584.349339:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203584.349340:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203584.349343:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203584.349347:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25270 00000400:00000200:2.0:1788203584.349350:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203584.349352:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203584.349354:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203584.349356:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349358:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.349359:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349361:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203584.349362:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203584.349363:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203584.349364:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203584.349366:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203584.349367:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203584.349368:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203584.349370:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349372:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203584.349373:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349374:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203584.349376:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203584.349378:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203584.349379:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203584.349380:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203584.349384:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203584.349385:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.349386:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203584.349388:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203584.349390:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.349391:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203584.349394:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203584.349396:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203584.349398:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203584.349399:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203584.349400:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203584.349401:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203584.349403:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203584.349405:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203584.349409:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203584.349412:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543321856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203584.349417:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203584.349419:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203584.349421:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543321856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203584.349427:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203584.349432:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25280) 00000800:00000200:3.1:1788203584.349799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203584.349801:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203584.350017:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203584.350019:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203584.350272:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203584.350274:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203584.350489:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203584.350491:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788203585.180243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203585.180255:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203585.180259:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203585.180269:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203585.180280:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203585.180286:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203585.180294:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203585.180303:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203585.180326:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203585.692231:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319944454, at 4319944512 00000100:00000010:2.0:1788203585.692250:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc7e0. 00000100:00000001:2.0:1788203586.204210:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203586.204220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203586.204225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203586.204235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203586.204240:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203586.204244:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203586.204250:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203586.204254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203586.204259:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203587.228188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203587.228199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203587.228201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203587.228208:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203587.228214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203587.228220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203587.228226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203587.228231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203587.228238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203588.252227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203588.252238:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203588.252241:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203588.252247:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203588.252265:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203588.252270:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203588.252274:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203588.252278:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203588.252282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203589.276231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203589.276251:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203589.276257:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203589.276265:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203589.276273:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203589.276279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203589.276287:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203589.276292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203589.276300:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203589.468998:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203589.469005:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203589.469718:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203589.469721:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203589.470014:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203589.470016:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203589.470361:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203589.470364:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203589.470686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203589.470689:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788203590.300253:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203590.300267:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203590.300271:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203590.300277:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203590.300282:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203590.300288:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203590.300291:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203590.300296:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203590.300300:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203590.812258:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319949574, at 4319949632 00000100:00000010:2.0:1788203590.812285:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc4e0. 00000100:00000001:2.0:1788203591.324266:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203591.324279:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203591.324286:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203591.324297:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203591.324308:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203591.324315:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203591.324324:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203591.324331:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203591.324340:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203592.348213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203592.348227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203592.348232:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203592.348243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203592.348251:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203592.348257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203592.348265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203592.348271:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203592.348278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203593.372271:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203593.372283:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203593.372287:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203593.372295:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203593.372300:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203593.372306:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203593.372310:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203593.372314:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203593.372319:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203594.396179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203594.396189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203594.396192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203594.396199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203594.396206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203594.396213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203594.396218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203594.396223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203594.396231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203594.588782:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203594.588788:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203594.589300:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203594.589302:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203594.589492:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203594.589494:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203594.589656:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203594.589658:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203594.589846:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203594.589848:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1788203595.420211:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203595.420227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203595.420233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203595.420250:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203595.420265:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203595.420272:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203595.420279:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203595.420285:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203595.420291:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203595.932225:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319954693, at 4319954752 00000100:00000010:2.0:1788203595.932241:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc240. 00000100:00000001:2.0:1788203596.444213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203596.444225:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203596.444230:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203596.444237:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203596.444248:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203596.444255:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203596.444266:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203596.444271:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203596.444278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203597.468224:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203597.468240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203597.468245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203597.468254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203597.468265:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203597.468273:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203597.468280:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203597.468284:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203597.468294:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203598.492210:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203598.492221:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203598.492227:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203598.492241:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203598.492253:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203598.492259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203598.492268:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203598.492273:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203598.492279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203599.516173:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203599.516183:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203599.516185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203599.516192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203599.516197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203599.516202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203599.516204:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203599.516208:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203599.516213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203599.708806:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203599.708811:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203599.709756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203599.709757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203599.709986:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203599.709987:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203599.710251:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203599.710253:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203599.710585:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203599.710587:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1788203600.540185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203600.540193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203600.540197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203600.540203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203600.540210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203600.540215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203600.540222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203600.540225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203600.540227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203601.052212:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319959813, at 4319959872 00000100:00000010:2.0:1788203601.052231:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc900. 00000100:00000001:2.0:1788203601.564164:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203601.564177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203601.564180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203601.564185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203601.564193:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203601.564201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203601.564207:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203601.564215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203601.564227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203602.588269:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203602.588280:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203602.588286:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203602.588303:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203602.588312:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203602.588320:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203602.588336:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203602.588341:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203602.588346:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203603.612152:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203603.612162:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203603.612164:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203603.612172:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203603.612179:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203603.612183:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203603.612187:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203603.612191:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203603.612196:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203604.636209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203604.636220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203604.636228:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203604.636242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203604.636249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203604.636256:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203604.636271:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203604.636275:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203604.636281:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203604.828705:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203604.828710:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203604.829408:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203604.829412:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203604.829656:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203604.829657:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203604.829918:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203604.829919:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203604.830144:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203604.830146:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000040:2.0:1788203606.172208:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319964933, at 4319964992 00000100:00000010:2.0:1788203606.172234:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc540. 00000020:00000001:2.0:1788203606.684303:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203606.684308:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203606.684312:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203607.708283:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203607.708286:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203607.708288:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203609.756294:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203609.756298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203609.756302:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203609.948263:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203609.948272:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203609.948274:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203609.948278:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203609.948281:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203609.948287:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25296 00000400:00000200:2.0:1788203609.948296:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203609.948301:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203609.948303:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203609.948306:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948308:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.948310:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948312:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203609.948315:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203609.948316:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203609.948318:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203609.948321:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203609.948322:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203609.948324:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203609.948327:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948328:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.948330:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948331:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203609.948333:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203609.948335:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948336:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203609.948338:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203609.948341:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203609.948343:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948345:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203609.948347:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203609.948356:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788203609.948358:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203609.948361:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203609.948363:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948366:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203609.948368:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203609.948372:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203609.948378:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203609.948384:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203609.948390:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543324544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203609.948398:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203609.948402:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203609.948406:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543324544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203609.948426:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203609.948428:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203609.948431:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203609.948432:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203609.948433:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203609.948435:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203609.948437:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25296 00000400:00000200:2.0:1788203609.948440:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203609.948441:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203609.948443:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203609.948445:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948447:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.948448:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948450:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203609.948451:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203609.948452:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203609.948454:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203609.948456:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203609.948457:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203609.948459:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203609.948462:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948464:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.948465:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948466:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203609.948468:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203609.948473:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948476:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203609.948480:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203609.948485:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203609.948489:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948491:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203609.948494:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203609.948500:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948503:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203609.948507:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788203609.948511:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203609.948515:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203609.948516:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948521:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203609.948522:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203609.948525:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203609.948530:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203609.948535:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203609.948547:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543324672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203609.948567:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203609.948572:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203609.948580:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543324672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203609.948599:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203609.948606:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203609.948611:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203609.948614:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203609.948616:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203609.948618:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203609.948621:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25296 00000400:00000200:2.0:1788203609.948628:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203609.948631:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203609.948635:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203609.948639:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948642:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.948645:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948648:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203609.948652:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203609.948653:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203609.948655:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203609.948658:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203609.948662:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203609.948667:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203609.948670:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948672:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.948673:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948674:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203609.948677:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203609.948682:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948685:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203609.948687:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203609.948692:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203609.948696:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948698:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203609.948701:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203609.948704:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948706:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203609.948711:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788203609.948714:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203609.948717:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203609.948719:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948723:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203609.948726:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203609.948727:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203609.948730:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203609.948734:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203609.948741:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543324800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203609.948754:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203609.948759:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203609.948765:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543324800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203609.948771:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203609.948773:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203609.948777:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203609.948778:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203609.948781:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203609.948787:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203609.948791:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25296 00000400:00000200:2.0:1788203609.948822:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203609.948829:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203609.948831:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203609.948835:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948840:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.948842:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948844:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203609.948849:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203609.948850:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203609.948854:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203609.948857:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203609.948859:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203609.948865:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203609.948868:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948869:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.948871:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.948872:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203609.948873:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203609.948875:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000200:3.1:1788203609.948892:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203609.948896:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788203609.948900:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203609.948904:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203609.948909:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203609.948913:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948914:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203609.948917:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203609.948920:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948924:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203609.948929:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203609.948934:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203609.948939:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203609.948940:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.948943:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203609.948945:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203609.948947:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203609.948953:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203609.948958:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203609.948966:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543324928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203609.948993:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203609.948997:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203609.949003:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543324928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203609.949010:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203609.949012:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203609.949017:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203609.949020:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203609.949023:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203609.949026:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203609.949030:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25296 00000400:00000200:2.0:1788203609.949034:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203609.949039:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203609.949041:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203609.949043:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.949045:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.949048:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.949051:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203609.949053:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203609.949054:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203609.949055:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203609.949057:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203609.949061:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203609.949064:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203609.949070:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.949073:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203609.949077:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203609.949079:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203609.949083:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203609.949088:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203609.949126:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203609.949130:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203609.949134:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203609.949139:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.949142:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203609.949144:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203609.949147:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.949150:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203609.949155:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203609.949160:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203609.949164:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203609.949166:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203609.949169:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203609.949172:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203609.949175:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203609.949180:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203609.949185:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203609.949194:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543325056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203609.949202:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203609.949205:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203609.949210:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543325056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203609.949215:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203609.949219:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25306) 00000800:00000200:3.1:1788203609.949415:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203609.949417:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203609.949603:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203609.949605:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203609.949841:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203609.949842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203609.950117:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203609.950120:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1788203610.780283:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203610.780287:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203610.780289:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203611.292209:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319970053, at 4319970112 00000100:00000010:2.0:1788203611.292224:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc120. 00000020:00000001:2.0:1788203612.828298:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203612.828303:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203612.828307:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203613.852308:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203613.852311:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203613.852314:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203614.876293:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203614.876297:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203614.876301:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203615.068161:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203615.068169:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203615.068172:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203615.068175:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203615.068178:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203615.068183:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25301 00000400:00000200:2.0:1788203615.068191:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203615.068196:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203615.068204:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203615.068207:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068209:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068211:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068213:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203615.068216:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203615.068217:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203615.068220:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203615.068222:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203615.068224:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203615.068226:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203615.068228:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068229:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068230:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068232:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203615.068235:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203615.068237:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068239:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203615.068241:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203615.068244:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203615.068246:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068247:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203615.068249:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203615.068257:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203615.068260:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203615.068263:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203615.068264:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068267:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203615.068268:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203615.068272:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203615.068278:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203615.068284:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203615.068290:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543325184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203615.068298:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203615.068302:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203615.068305:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543325184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203615.068323:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203615.068325:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203615.068328:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203615.068329:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203615.068330:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203615.068332:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203615.068334:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25301 00000400:00000200:2.0:1788203615.068336:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203615.068339:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203615.068340:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203615.068342:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068344:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068345:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068347:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203615.068348:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203615.068349:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203615.068350:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203615.068352:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203615.068353:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203615.068354:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203615.068357:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068358:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068360:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068362:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203615.068364:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203615.068366:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068368:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203615.068369:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203615.068375:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203615.068380:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068385:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203615.068391:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203615.068398:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068403:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203615.068409:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203615.068416:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203615.068423:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203615.068427:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068431:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203615.068435:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203615.068437:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203615.068442:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203615.068448:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203615.068455:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543325312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203615.068468:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203615.068471:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203615.068475:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543325312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203615.068485:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203615.068488:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203615.068493:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203615.068496:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203615.068499:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203615.068501:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203615.068505:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25301 00000400:00000200:2.0:1788203615.068510:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203615.068514:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203615.068518:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203615.068523:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068528:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068533:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068537:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203615.068539:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203615.068543:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203615.068544:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203615.068546:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203615.068550:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203615.068552:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203615.068559:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068561:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068564:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068566:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203615.068567:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203615.068569:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068570:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203615.068571:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203615.068575:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203615.068578:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068579:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203615.068581:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203615.068586:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068590:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203615.068593:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203615.068596:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203615.068600:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203615.068603:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068606:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203615.068607:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203615.068610:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203615.068615:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203615.068620:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203615.068626:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543325440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203615.068643:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203615.068646:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203615.068653:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543325440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203615.068668:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203615.068671:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203615.068676:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203615.068678:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203615.068679:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203615.068682:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203615.068686:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25301 00000400:00000200:2.0:1788203615.068694:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203615.068700:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203615.068703:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1788203615.068706:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788203615.068709:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:3.1:1788203615.068711:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788203615.068712:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068716:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068719:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203615.068722:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203615.068724:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203615.068729:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203615.068730:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203615.068733:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203615.068739:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203615.068746:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068748:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068752:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068755:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203615.068759:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203615.068763:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068767:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203615.068772:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203615.068775:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203615.068780:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068782:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203615.068784:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203615.068787:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068803:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203615.068806:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203615.068808:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203615.068810:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203615.068812:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068814:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203615.068817:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203615.068819:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203615.068825:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203615.068831:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203615.068843:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543325568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203615.068853:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203615.068858:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203615.068867:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543325568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203615.068879:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203615.068882:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203615.068885:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203615.068889:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203615.068892:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203615.068896:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203615.068900:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25301 00000400:00000200:2.0:1788203615.068907:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203615.068913:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203615.068918:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203615.068923:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068934:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068937:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068939:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203615.068941:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203615.068942:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203615.068943:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203615.068945:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203615.068949:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203615.068954:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203615.068957:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068960:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203615.068964:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068966:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203615.068970:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203615.068974:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203615.068979:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203615.068982:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203615.068986:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203615.068988:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068990:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203615.068992:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203615.068993:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.068994:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203615.068997:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203615.068999:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203615.069001:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203615.069004:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203615.069013:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203615.069016:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203615.069018:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203615.069024:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203615.069030:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203615.069039:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543325696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203615.069066:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203615.069070:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203615.069078:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543325696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203615.069109:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203615.069114:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25311) 00000800:00000200:3.1:1788203615.069314:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203615.069316:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203615.069514:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203615.069517:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203615.069721:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203615.069723:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203615.069958:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203615.069961:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1788203615.900334:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203615.900338:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203615.900342:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203616.412212:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319975173, at 4319975232 00000100:00000010:2.0:1788203616.412225:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcc60. 00000020:00000001:2.0:1788203616.924393:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203616.924401:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203616.924406:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203617.948324:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203617.948335:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203617.948351:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203618.972442:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203618.972447:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203618.972452:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203619.996373:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203619.996379:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203619.996383:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203620.188243:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203620.188255:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203620.188267:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203620.188271:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203620.188276:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203620.188285:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25306 00000400:00000200:2.0:1788203620.188297:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203620.188306:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203620.188309:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203620.188313:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188316:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.188318:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188321:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203620.188333:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203620.188336:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203620.188339:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203620.188343:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203620.188346:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203620.188349:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203620.188353:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188355:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.188358:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188360:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203620.188364:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203620.188368:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188371:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203620.188375:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203620.188380:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203620.188383:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.188386:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203620.188389:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203620.188401:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788203620.188404:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203620.188409:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203620.188410:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.188413:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203620.188416:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203620.188421:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203620.188429:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203620.188438:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203620.188450:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543325824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203620.188462:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203620.188468:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203620.188474:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543325824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203620.188503:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203620.188506:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203620.188509:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203620.188510:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203620.188511:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203620.188514:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203620.188516:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25306 00000400:00000200:2.0:1788203620.188520:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203620.188523:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203620.188524:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203620.188527:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188528:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.188530:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188532:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203620.188534:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203620.188535:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203620.188536:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203620.188538:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203620.188540:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203620.188541:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203620.188543:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188545:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.188546:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188549:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203620.188551:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203620.188554:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188556:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203620.188557:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203620.188564:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203620.188567:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.188569:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203620.188575:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203620.188579:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.188581:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203620.188585:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788203620.188588:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203620.188592:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203620.188594:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.188598:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203620.188600:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203620.188603:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203620.188609:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203620.188613:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203620.188621:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543325952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203620.188630:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203620.188635:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203620.188641:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543325952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203620.188657:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203620.188662:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203620.188666:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203620.188670:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203620.188674:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203620.188678:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203620.188687:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25306 00000400:00000200:2.0:1788203620.188695:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203620.188713:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203620.188718:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203620.188723:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188726:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.188729:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188736:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203620.188738:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203620.188739:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203620.188742:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203620.188745:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203620.188749:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203620.188754:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203620.188761:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188763:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.188767:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188772:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203620.188778:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203620.188784:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188787:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203620.188793:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203620.188796:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203620.188801:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.188803:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203620.188807:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203620.188810:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.188815:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203620.188822:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788203620.188834:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203620.188843:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203620.188846:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.188852:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203620.188855:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203620.188860:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203620.188869:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203620.188878:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203620.188886:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543326080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203620.188901:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203620.188907:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203620.188911:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543326080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203620.188920:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203620.188925:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203620.188931:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203620.188934:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203620.188936:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203620.188940:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203620.188945:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25306 00000800:00000200:3.1:1788203620.188951:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:2.0:1788203620.188952:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203620.188958:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:3.1:1788203620.188959:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00080000:2.0:1788203620.188970:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203620.188977:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188982:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.188983:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.188985:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203620.188986:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203620.188989:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203620.188994:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203620.188996:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203620.189000:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203620.189004:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203620.189008:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.189012:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.189014:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.189020:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203620.189024:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203620.189029:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203620.189035:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203620.189039:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203620.189044:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203620.189050:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.189054:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203620.189058:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203620.189063:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.189070:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203620.189080:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203620.189085:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203620.189088:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203620.189113:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.189115:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203620.189116:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203620.189118:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203620.189122:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203620.189126:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203620.189132:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543326208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203620.189146:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203620.189149:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203620.189153:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543326208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203620.189161:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203620.189164:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203620.189172:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203620.189174:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203620.189176:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203620.189182:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203620.189190:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25306 00000400:00000200:2.0:1788203620.189198:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203620.189206:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203620.189212:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203620.189233:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.189243:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.189248:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.189251:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203620.189253:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203620.189255:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203620.189257:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203620.189259:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203620.189261:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203620.189264:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203620.189267:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.189269:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203620.189271:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203620.189274:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203620.189277:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203620.189280:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203620.189283:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203620.189285:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203620.189289:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203620.189292:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.189294:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203620.189296:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203620.189299:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.189301:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203620.189306:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203620.189309:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203620.189312:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203620.189313:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203620.189315:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203620.189317:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203620.189319:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203620.189324:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203620.189328:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203620.189333:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543326336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203620.189351:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203620.189357:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203620.189367:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543326336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203620.189387:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203620.189397:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25316) 00000800:00000200:3.1:1788203620.189672:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203620.189684:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203620.189913:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203620.189915:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203620.190184:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203620.190186:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203620.190369:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203620.190371:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1788203621.020381:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203621.020387:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203621.020391:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203621.532252:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319980294, at 4319980352 00000100:00000010:2.0:1788203621.532266:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc840. 00000020:00000001:2.0:1788203622.044374:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203622.044379:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203622.044381:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203623.068381:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203623.068388:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203623.068392:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203624.092321:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203624.092326:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203624.092328:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203625.116268:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203625.116272:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203625.116275:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1788203625.308764:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203625.308768:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203625.309571:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203625.309573:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203625.309791:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203625.309793:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203625.310005:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203625.310007:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203625.310305:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203625.310307:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1788203626.140348:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203626.140352:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203626.140355:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203626.652223:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319985413, at 4319985472 00000100:00000010:2.0:1788203626.652249:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa2a0. 00000020:00000001:2.0:1788203627.164375:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203627.164381:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203627.164386:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203628.188303:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203628.188308:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203628.188312:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203629.212387:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203629.212391:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203629.212393:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203630.236281:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203630.236285:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203630.236288:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1788203630.428644:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203630.428648:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203630.429088:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203630.429109:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203630.429428:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203630.429431:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203630.429643:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203630.429645:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203630.429817:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203630.429819:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1788203631.260320:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203631.260325:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203631.260330:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203631.772164:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319990533, at 4319990592 00000100:00000010:2.0:1788203631.772180:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa540. 00000020:00000001:2.0:1788203632.284614:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203632.284618:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203632.284622:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203633.308307:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203633.308311:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203633.308314:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203634.332320:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203634.332325:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203634.332328:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203635.356359:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203635.356363:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203635.356365:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203635.548265:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203635.548276:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203635.548279:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203635.548283:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203635.548288:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203635.548295:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25321 00000400:00000200:2.0:1788203635.548305:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203635.548314:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203635.548318:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203635.548327:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548330:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.548333:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548336:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203635.548341:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203635.548343:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203635.548347:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203635.548351:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203635.548353:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203635.548356:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203635.548360:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548362:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.548364:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548367:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203635.548370:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203635.548374:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548377:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203635.548381:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203635.548386:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203635.548389:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.548391:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203635.548395:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203635.548409:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203635.548414:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203635.548418:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203635.548420:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.548423:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203635.548426:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203635.548430:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203635.548461:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203635.548469:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203635.548480:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543327744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203635.548492:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203635.548498:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203635.548503:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543327744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203635.548525:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203635.548528:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203635.548531:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203635.548535:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203635.548536:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203635.548539:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203635.548541:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25321 00000400:00000200:2.0:1788203635.548544:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203635.548546:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203635.548548:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203635.548551:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548552:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.548554:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548556:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203635.548558:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203635.548559:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203635.548560:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203635.548562:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203635.548563:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203635.548564:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203635.548566:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548568:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.548569:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548571:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203635.548574:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203635.548583:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548588:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203635.548592:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203635.548599:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203635.548605:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.548609:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203635.548616:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203635.548622:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.548626:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203635.548631:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203635.548635:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203635.548640:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203635.548643:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.548647:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203635.548649:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203635.548653:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203635.548657:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203635.548662:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203635.548670:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543327872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203635.548677:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203635.548680:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203635.548684:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543327872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203635.548703:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203635.548706:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203635.548713:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203635.548714:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203635.548715:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203635.548718:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203635.548721:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25321 00000400:00000200:2.0:1788203635.548727:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203635.548731:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203635.548733:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203635.548738:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548741:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.548742:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548744:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203635.548745:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203635.548746:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203635.548747:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203635.548752:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203635.548756:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203635.548761:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203635.548766:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548772:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.548778:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548782:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203635.548787:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203635.548793:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203635.548798:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203635.548804:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203635.548809:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203635.548812:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.548817:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203635.548821:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203635.548826:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.548830:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203635.548835:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203635.548839:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203635.548844:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203635.548848:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.548852:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203635.548856:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203635.548860:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203635.548875:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203635.548882:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203635.548890:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543328000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203635.548904:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203635.548907:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203635.548916:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543328000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1788203635.548934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203635.548939:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203635.548940:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000100:00080000:2.0:1788203635.548945:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203635.548953:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203635.548958:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203635.548962:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203635.548968:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203635.548976:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25321 00000400:00000200:2.0:1788203635.548984:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203635.548998:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203635.549003:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203635.549010:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549014:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.549016:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549018:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203635.549020:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203635.549021:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203635.549023:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203635.549026:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203635.549030:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203635.549034:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203635.549043:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549047:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.549052:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549063:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203635.549068:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203635.549075:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549081:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203635.549083:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203635.549120:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203635.549124:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.549127:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203635.549131:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203635.549135:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.549141:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203635.549148:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203635.549161:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203635.549168:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203635.549171:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.549176:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203635.549178:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203635.549182:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203635.549190:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203635.549196:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203635.549207:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543328128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203635.549220:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203635.549224:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203635.549232:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543328128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203635.549253:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203635.549259:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203635.549264:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203635.549265:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203635.549267:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203635.549272:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203635.549278:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25321 00000400:00000200:2.0:1788203635.549288:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203635.549295:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203635.549299:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203635.549307:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549311:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.549317:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549322:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203635.549324:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203635.549326:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203635.549331:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203635.549336:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203635.549341:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203635.549345:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203635.549352:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549358:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203635.549365:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549370:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203635.549376:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203635.549383:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203635.549391:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203635.549395:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203635.549400:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203635.549407:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.549412:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203635.549416:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203635.549422:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.549426:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203635.549445:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203635.549450:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203635.549455:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203635.549459:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203635.549464:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203635.549467:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203635.549471:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203635.549477:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203635.549484:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203635.549493:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543328256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203635.549502:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203635.549507:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203635.549516:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543328256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203635.549539:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203635.549548:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25332) 00000800:00000200:0.1:1788203635.549663:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203635.549666:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203635.549954:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203635.549956:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203635.550247:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203635.550250:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203635.550643:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203635.550646:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000020:00000001:2.0:1788203636.380300:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203636.380305:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203636.380308:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203636.892208:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4319995653, at 4319995712 00000100:00000010:2.0:1788203636.892221:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faa20. 00000020:00000001:2.0:1788203637.405639:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203637.405644:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203637.405648:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203638.428320:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203638.428324:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203638.428328:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203639.452396:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203639.452403:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203639.452407:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203640.476302:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203640.476308:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203640.476312:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788203640.669013:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203640.669605:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203640.669843:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203640.670000:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203640.670412:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788203641.500299:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203641.500308:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203641.500318:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203642.012190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320000773, at 4320000832 00000100:00000010:2.0:1788203642.012208:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fce40. 00000020:00000001:2.0:1788203642.524341:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203642.524349:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203642.524353:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203643.548348:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203643.548352:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203643.548354:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203644.572285:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203644.572289:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203644.572292:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203645.596276:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203645.596279:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203645.596282:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788203645.789496:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203645.789501:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203645.790182:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203645.790185:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203645.790475:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203645.790477:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203645.790741:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203645.790744:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203645.791863:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203645.791866:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1788203646.620529:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203646.620536:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203646.620541:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203647.132150:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320005894, at 4320005952 00000100:00000010:2.0:1788203647.132161:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc0c0. 00000020:00000001:2.0:1788203647.644283:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203647.644287:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203647.644289:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203648.668299:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203648.668304:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203648.668307:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203649.692287:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203649.692291:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203649.692293:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203650.716279:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203650.716283:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203650.716285:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788203650.908770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203650.908776:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203650.909342:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203650.909343:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203650.909520:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203650.909521:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203650.909678:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203650.909680:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203650.909829:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203650.909830:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000020:00000001:2.0:1788203651.740302:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203651.740305:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203651.740308:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203652.252180:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320011013, at 4320011072 00000100:00000010:2.0:1788203652.252198:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc720. 00000020:00000001:2.0:1788203652.764411:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203652.764417:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203652.764421:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203653.788370:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203653.788374:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203653.788376:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203654.812417:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203654.812425:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203654.812430:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203655.836353:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203655.836358:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203655.836363:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203656.028272:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203656.028287:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203656.028290:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203656.028295:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203656.028300:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203656.028311:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25342 00000400:00000200:2.0:1788203656.028324:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203656.028333:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203656.028337:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203656.028341:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028345:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.028348:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028352:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203656.028358:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203656.028360:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203656.028363:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203656.028368:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203656.028371:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203656.028374:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203656.028377:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028379:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.028382:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028384:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203656.028388:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203656.028392:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028395:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203656.028398:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203656.028402:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203656.028406:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.028410:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203656.028413:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203656.028426:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788203656.028431:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203656.028435:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203656.028438:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.028442:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203656.028445:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203656.028451:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203656.028460:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203656.028469:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203656.028480:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543330304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203656.028493:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203656.028498:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203656.028504:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543330304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203656.028531:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203656.028535:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203656.028539:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203656.028541:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203656.028543:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203656.028546:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203656.028550:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25342 00000400:00000200:2.0:1788203656.028555:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203656.028559:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203656.028562:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203656.028566:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028569:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.028571:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028574:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203656.028580:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203656.028582:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203656.028586:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203656.028591:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203656.028595:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203656.028601:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203656.028610:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028615:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.028620:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028624:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203656.028628:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203656.028634:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028640:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203656.028644:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203656.028653:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203656.028659:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.028663:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203656.028668:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203656.028675:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.028680:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203656.028687:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788203656.028692:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203656.028695:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203656.028706:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.028711:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203656.028715:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203656.028719:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203656.028727:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203656.028733:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203656.028744:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543330432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203656.028761:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203656.028767:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203656.028781:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543330432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203656.028795:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203656.028801:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203656.028810:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203656.028813:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203656.028817:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203656.028824:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203656.028831:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25342 00000400:00000200:2.0:1788203656.028860:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203656.028869:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203656.028874:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203656.028880:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028885:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1788203656.028900:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788203656.028902:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788203656.028904:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788203656.028907:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203656.028914:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203656.028917:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203656.028922:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203656.028925:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203656.028930:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203656.028937:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203656.028944:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028949:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.028954:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028959:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203656.028965:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203656.028972:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203656.028977:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203656.028981:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203656.028986:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203656.028992:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.028995:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203656.029000:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203656.029003:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.029006:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203656.029012:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788203656.029017:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203656.029022:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203656.029024:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.029029:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203656.029032:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203656.029034:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203656.029040:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203656.029046:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203656.029057:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543330560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203656.029134:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203656.029140:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203656.029151:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543330560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203656.029182:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203656.029188:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203656.029199:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203656.029203:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203656.029208:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203656.029215:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203656.029221:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25342 00000400:00000200:2.0:1788203656.029231:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203656.029241:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203656.029247:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203656.029253:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029259:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.029266:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029271:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203656.029277:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203656.029281:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203656.029286:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203656.029290:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203656.029295:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203656.029301:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203656.029310:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029314:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.029321:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029326:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203656.029332:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203656.029340:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029345:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203656.029349:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203656.029354:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203656.029361:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.029367:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203656.029371:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203656.029377:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.029382:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203656.029389:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788203656.029393:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203656.029396:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203656.029400:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.029404:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203656.029408:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203656.029412:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203656.029420:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203656.029427:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203656.029437:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543330688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203656.029453:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203656.029457:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203656.029465:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543330688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203656.029489:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203656.029495:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203656.029502:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203656.029532:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203656.029537:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203656.029544:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203656.029551:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25342 00000400:00000200:2.0:1788203656.029558:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203656.029566:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203656.029570:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203656.029577:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029581:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.029587:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029592:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203656.029596:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203656.029599:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203656.029603:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203656.029607:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203656.029612:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203656.029618:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203656.029626:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029632:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203656.029637:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029642:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203656.029647:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203656.029655:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203656.029661:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203656.029667:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203656.029672:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203656.029677:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.029681:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203656.029685:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203656.029691:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.029696:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203656.029696:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203656.029703:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000010:2.0:1788203656.029704:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203656.029709:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203656.029716:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203656.029720:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203656.029726:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203656.029728:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203656.029732:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203656.029739:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203656.029747:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203656.029758:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543330816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203656.029776:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203656.029781:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203656.029791:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543330816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203656.029812:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203656.029819:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25352) 00000800:00000200:3.1:1788203656.029986:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203656.029988:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203656.030270:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203656.030272:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203656.030509:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203656.030511:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1788203656.860418:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203656.860424:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203656.860428:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203657.372195:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320016133, at 4320016192 00000100:00000010:2.0:1788203657.372219:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc180. 00000020:00000001:2.0:1788203657.884360:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203657.884365:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203657.884369:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203658.908381:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203658.908386:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203658.908390:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203659.932275:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203659.932280:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203659.932284:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203660.956326:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203660.956330:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203660.956333:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788203661.148785:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203661.148791:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203661.149499:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203661.149500:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203661.149800:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203661.149809:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203661.150054:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203661.150056:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203661.150276:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203661.150278:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000020:00000001:2.0:1788203661.980320:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203661.980323:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203661.980326:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203662.492182:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320021253, at 4320021312 00000100:00000010:2.0:1788203662.492199:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcae0. 00000020:00000001:2.0:1788203663.004362:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203663.004368:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203663.004372:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203664.028366:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203664.028372:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203664.028376:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203665.052361:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203665.052365:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203665.052368:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203666.076379:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203666.076384:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203666.076388:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203666.268288:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203666.268300:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203666.268303:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203666.268307:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203666.268312:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203666.268320:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25352 00000400:00000200:2.0:1788203666.268330:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203666.268338:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203666.268341:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203666.268345:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268349:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268351:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268353:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203666.268357:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203666.268359:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203666.268361:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203666.268364:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203666.268365:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203666.268367:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203666.268369:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268370:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268371:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268373:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203666.268375:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203666.268377:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268378:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203666.268380:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203666.268383:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203666.268385:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268386:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203666.268389:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203666.268396:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203666.268399:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203666.268402:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203666.268403:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268406:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203666.268408:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203666.268412:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203666.268418:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203666.268423:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203666.268430:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543331584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203666.268438:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203666.268442:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203666.268445:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543331584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203666.268470:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203666.268474:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203666.268477:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203666.268479:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203666.268480:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203666.268484:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203666.268487:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25352 00000400:00000200:2.0:1788203666.268491:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203666.268494:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203666.268497:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203666.268499:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268502:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268504:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268506:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203666.268509:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203666.268510:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203666.268512:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203666.268515:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203666.268517:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203666.268519:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203666.268522:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268524:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268526:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268528:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203666.268531:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203666.268534:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268537:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203666.268539:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203666.268545:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203666.268547:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268549:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203666.268553:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203666.268557:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268559:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203666.268562:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203666.268564:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203666.268567:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203666.268568:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268570:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203666.268571:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203666.268573:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203666.268576:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203666.268580:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203666.268585:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543331712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203666.268591:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203666.268593:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203666.268612:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543331712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203666.268629:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203666.268632:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203666.268635:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203666.268637:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203666.268638:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203666.268640:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203666.268643:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25352 00000400:00000200:2.0:1788203666.268690:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203666.268693:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203666.268695:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203666.268698:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268701:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268703:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268705:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203666.268708:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203666.268709:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203666.268711:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203666.268713:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203666.268715:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203666.268717:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203666.268720:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268722:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268724:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268727:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788203666.268727:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:2.0:1788203666.268729:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000800:00000200:3.1:1788203666.268731:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00080000:2.0:1788203666.268732:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268735:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203666.268737:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203666.268740:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203666.268742:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268744:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203666.268746:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203666.268748:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268751:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203666.268755:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203666.268757:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203666.268760:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203666.268761:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268763:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203666.268764:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203666.268766:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203666.268771:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203666.268774:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203666.268778:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543331840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203666.268787:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203666.268789:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203666.268793:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543331840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203666.268800:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203666.268803:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203666.268807:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203666.268808:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203666.268810:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203666.268813:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203666.268815:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25352 00000400:00000200:2.0:1788203666.268818:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203666.268821:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203666.268832:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203666.268836:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268838:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268840:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268842:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203666.268844:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203666.268846:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203666.268848:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203666.268850:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203666.268852:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203666.268854:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203666.268857:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268859:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268861:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268864:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203666.268867:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203666.268870:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268873:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203666.268875:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203666.268877:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203666.268880:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268882:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203666.268884:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203666.268887:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268889:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203666.268892:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203666.268894:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203666.268897:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203666.268898:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.268911:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203666.268912:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203666.268914:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203666.268918:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203666.268922:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203666.268935:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543331968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203666.268943:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203666.268945:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203666.268950:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543331968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203666.268958:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203666.268961:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203666.268965:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203666.268966:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203666.268968:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203666.268971:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203666.268974:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25352 00000400:00000200:2.0:1788203666.268977:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203666.268980:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203666.268983:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203666.268986:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268988:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.268991:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.268993:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203666.268995:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203666.268996:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203666.268998:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203666.269000:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203666.269002:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203666.269004:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203666.269007:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.269010:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203666.269012:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203666.269014:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203666.269016:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203666.269019:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203666.269021:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203666.269023:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203666.269026:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203666.269028:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.269030:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203666.269032:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203666.269034:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.269036:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203666.269039:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203666.269042:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203666.269044:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203666.269045:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203666.269048:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203666.269049:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203666.269051:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203666.269054:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203666.269058:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203666.269068:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543332096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203666.269076:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203666.269078:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203666.269082:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543332096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203666.269140:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203666.269145:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25362) 00000800:00000200:3.1:1788203666.269241:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203666.269244:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203666.269460:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203666.269462:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203666.269712:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203666.269714:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203666.270142:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203666.270145:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000020:00000001:2.0:1788203667.100341:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203667.100344:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203667.100347:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203667.612184:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320026373, at 4320026432 00000100:00000010:2.0:1788203667.612193:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc300. 00000020:00000001:2.0:1788203668.124320:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203668.124325:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203668.124328:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203669.148381:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203669.148387:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203669.148391:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203670.172339:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203670.172346:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203670.172352:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203671.196346:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203671.196351:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203671.196355:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203671.388199:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203671.388207:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203671.388209:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203671.388212:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203671.388215:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203671.388220:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25357 00000400:00000200:2.0:1788203671.388229:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203671.388234:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203671.388236:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203671.388239:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388241:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.388242:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388244:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203671.388247:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203671.388248:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203671.388250:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203671.388253:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203671.388254:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203671.388256:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203671.388258:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388259:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.388260:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388262:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203671.388263:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203671.388265:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388267:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203671.388269:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203671.388271:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203671.388273:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388274:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203671.388276:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203671.388283:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203671.388285:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203671.388289:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203671.388290:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388292:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203671.388294:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203671.388297:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203671.388303:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203671.388308:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203671.388315:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543332224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203671.388322:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203671.388325:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203671.388328:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543332224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203671.388345:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203671.388347:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203671.388350:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203671.388351:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203671.388352:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203671.388354:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203671.388356:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25357 00000400:00000200:2.0:1788203671.388358:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203671.388360:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203671.388361:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203671.388363:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388365:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.388367:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388368:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203671.388370:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203671.388372:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203671.388373:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203671.388375:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203671.388378:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203671.388383:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203671.388389:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388394:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.388400:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388407:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203671.388411:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203671.388417:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388424:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203671.388429:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203671.388436:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203671.388443:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388447:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203671.388458:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203671.388463:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388465:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203671.388468:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788203671.388472:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203671.388474:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203671.388477:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388479:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203671.388482:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203671.388485:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203671.388490:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203671.388496:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203671.388505:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543332352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203671.388514:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203671.388518:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203671.388522:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543332352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203671.388534:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203671.388538:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203671.388542:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203671.388544:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203671.388547:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203671.388550:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203671.388555:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25357 00000400:00000200:2.0:1788203671.388577:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203671.388583:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203671.388585:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203671.388589:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388592:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.388596:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388599:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203671.388603:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203671.388606:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000800:00000200:3.1:1788203671.388607:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788203671.388608:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203671.388611:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000800:00000200:3.1:1788203671.388612:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000020:00000001:2.0:1788203671.388613:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203671.388616:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203671.388621:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388625:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.388627:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388630:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203671.388633:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203671.388638:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388641:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203671.388645:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203671.388648:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203671.388651:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388654:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203671.388656:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203671.388661:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388665:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203671.388670:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788203671.388673:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203671.388677:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203671.388680:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388682:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203671.388685:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203671.388688:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203671.388693:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203671.388699:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203671.388705:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543332480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203671.388716:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203671.388721:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203671.388730:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543332480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203671.388744:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203671.388749:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203671.388757:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203671.388762:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203671.388766:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203671.388774:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203671.388778:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25357 00000400:00000200:2.0:1788203671.388786:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203671.388792:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203671.388796:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203671.388801:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388806:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.388825:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388830:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203671.388836:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203671.388837:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203671.388839:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203671.388842:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203671.388845:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203671.388848:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203671.388855:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388858:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.388863:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388867:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203671.388872:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203671.388877:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203671.388881:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203671.388886:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203671.388889:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203671.388894:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388896:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203671.388899:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203671.388905:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388909:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203671.388913:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788203671.388918:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203671.388924:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203671.388926:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.388930:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203671.388931:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203671.388932:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203671.388935:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203671.388939:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203671.388947:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543332608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203671.388962:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203671.388967:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203671.388973:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543332608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203671.388990:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203671.388996:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203671.389003:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203671.389006:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203671.389010:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203671.389016:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203671.389022:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25357 00000400:00000200:2.0:1788203671.389030:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203671.389037:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203671.389043:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203671.389049:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.389052:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.389057:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.389062:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203671.389065:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203671.389067:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203671.389070:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203671.389073:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203671.389076:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203671.389079:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203671.389085:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.389089:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203671.389107:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203671.389108:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203671.389110:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203671.389111:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203671.389112:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203671.389113:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203671.389115:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203671.389117:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.389118:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203671.389119:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203671.389121:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.389122:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203671.389125:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203671.389126:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203671.389128:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203671.389129:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203671.389130:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203671.389133:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203671.389137:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203671.389140:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203671.389146:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203671.389154:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543332736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203671.389165:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203671.389168:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203671.389173:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543332736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203671.389185:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203671.389189:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25367) 00000800:00000200:3.1:1788203671.389244:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203671.389246:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788203671.389436:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203671.389437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788203671.389589:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203671.389591:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788203671.389795:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203671.389797:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000020:00000001:2.0:1788203672.220318:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203672.220322:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203672.220324:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203672.732198:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320031493, at 4320031552 00000100:00000010:2.0:1788203672.732215:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf60. 00000020:00000001:2.0:1788203673.244377:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203673.244383:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203673.244388:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203674.268335:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203674.268339:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203674.268342:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203675.292285:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203675.292289:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203675.292291:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203676.316406:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203676.316409:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203676.316412:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:0.1:1788203676.508907:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203676.508913:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203676.509762:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203676.509765:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203676.510084:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203676.510086:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203676.510405:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203676.510407:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203676.510700:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203676.510703:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1788203677.340277:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203677.340282:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203677.340285:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203677.852182:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320036613, at 4320036672 00000100:00000010:2.0:1788203677.852201:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa360. 00000020:00000001:2.0:1788203678.364355:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203678.364360:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203678.364366:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203679.388391:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203679.388394:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203679.388397:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203680.412264:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203680.412268:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203680.412271:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203681.436339:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203681.436345:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203681.436349:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203681.628194:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203681.628207:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203681.628210:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203681.628215:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203681.628220:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203681.628229:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25368 00000400:00000200:2.0:1788203681.628241:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203681.628249:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203681.628253:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203681.628256:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628260:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.628263:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628265:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203681.628270:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203681.628272:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203681.628276:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203681.628280:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203681.628283:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203681.628286:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203681.628289:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628291:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.628293:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628296:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203681.628299:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203681.628302:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628305:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203681.628309:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203681.628313:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203681.628316:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628319:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203681.628322:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203681.628333:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203681.628337:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203681.628341:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203681.628342:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628346:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203681.628349:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203681.628354:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203681.628361:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203681.628369:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203681.628380:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543333504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203681.628391:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203681.628397:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203681.628402:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543333504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203681.628428:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203681.628431:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203681.628435:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203681.628436:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203681.628438:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203681.628441:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203681.628444:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25368 00000400:00000200:2.0:1788203681.628448:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203681.628452:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203681.628454:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203681.628458:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628461:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.628463:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628466:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203681.628468:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203681.628469:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203681.628472:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203681.628476:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203681.628479:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203681.628484:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203681.628490:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628494:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.628500:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628504:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203681.628509:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203681.628514:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628518:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203681.628522:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203681.628529:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203681.628535:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628540:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203681.628547:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203681.628552:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628557:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203681.628564:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203681.628570:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203681.628574:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203681.628578:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628581:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203681.628582:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203681.628585:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203681.628593:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203681.628598:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203681.628607:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543333632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203681.628617:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203681.628622:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203681.628626:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543333632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203681.628655:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203681.628658:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203681.628665:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203681.628669:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203681.628671:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203681.628676:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203681.628682:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25368 00000400:00000200:2.0:1788203681.628691:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203681.628698:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203681.628703:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203681.628708:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628712:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.628717:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628721:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203681.628726:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203681.628728:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203681.628730:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203681.628732:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203681.628736:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203681.628741:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203681.628747:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628752:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.628756:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628761:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203681.628766:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203681.628772:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628778:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203681.628782:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203681.628786:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203681.628792:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628794:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203681.628796:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203681.628802:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628806:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203681.628812:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203681.628816:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203681.628820:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203681.628822:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628823:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203681.628824:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203681.628826:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203681.628832:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203681.628835:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203681.628841:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543333760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203681.628849:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203681.628852:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203681.628857:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543333760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1788203681.628857:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203681.628867:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788203681.628868:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203681.628873:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203681.628878:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203681.628882:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203681.628884:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203681.628887:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203681.628890:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25368 00000400:00000200:2.0:1788203681.628895:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203681.628900:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203681.628902:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203681.628907:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628909:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.628913:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628916:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203681.628918:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203681.628919:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203681.628923:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203681.628924:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203681.628927:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203681.628931:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203681.628937:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628941:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.628943:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628945:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203681.628949:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203681.628953:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203681.628958:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203681.628961:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203681.628964:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203681.628967:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628971:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203681.628974:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203681.628976:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628979:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203681.628982:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203681.628986:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203681.628989:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203681.628991:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.628992:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203681.628994:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203681.628996:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203681.629000:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203681.629004:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203681.629010:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543333888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203681.629020:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203681.629023:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203681.629029:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543333888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203681.629040:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203681.629043:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203681.629047:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203681.629049:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203681.629052:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203681.629057:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203681.629060:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25368 00000400:00000200:2.0:1788203681.629067:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203681.629073:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203681.629079:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203681.629086:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.629128:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.629134:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.629141:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203681.629147:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203681.629151:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203681.629155:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203681.629159:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203681.629163:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203681.629168:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203681.629174:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.629179:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203681.629184:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203681.629189:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203681.629194:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203681.629200:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203681.629206:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203681.629211:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203681.629216:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203681.629220:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.629224:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203681.629228:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203681.629233:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.629236:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203681.629243:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203681.629248:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203681.629253:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203681.629256:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203681.629260:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203681.629264:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203681.629266:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203681.629272:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203681.629278:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203681.629288:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543334016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203681.629313:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203681.629317:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203681.629326:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543334016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203681.629340:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203681.629348:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25378) 00000800:00000200:0.1:1788203681.629480:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203681.629481:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203681.629670:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203681.629671:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203681.629841:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203681.629843:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203681.630060:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203681.630062:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000020:00000001:2.0:1788203682.460316:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203682.460321:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203682.460324:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203682.972225:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320041733, at 4320041792 00000100:00000010:2.0:1788203682.972252:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fab40. 00000020:00000001:2.0:1788203683.484347:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203683.484352:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203683.484356:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203684.508344:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203684.508348:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203684.508350:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203685.532330:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203685.532335:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203685.532340:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203686.556336:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203686.556341:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203686.556345:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203686.748235:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203686.748248:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203686.748251:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203686.748256:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203686.748261:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203686.748270:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25373 00000400:00000200:2.0:1788203686.748282:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203686.748290:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203686.748294:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203686.748298:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748301:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.748304:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748307:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203686.748311:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203686.748313:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203686.748316:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203686.748320:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203686.748322:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203686.748325:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203686.748328:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748330:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.748332:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748334:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203686.748336:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203686.748340:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748342:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203686.748345:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203686.748350:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203686.748352:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748355:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203686.748357:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203686.748369:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203686.748373:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203686.748377:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203686.748378:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748382:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203686.748385:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203686.748390:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203686.748398:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203686.748407:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203686.748417:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543334144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203686.748426:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203686.748429:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203686.748433:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543334144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203686.748452:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203686.748455:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203686.748457:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203686.748458:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203686.748459:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203686.748461:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203686.748463:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25373 00000400:00000200:2.0:1788203686.748466:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203686.748467:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203686.748469:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203686.748471:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748473:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.748475:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748476:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203686.748478:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203686.748479:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203686.748480:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203686.748481:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203686.748483:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203686.748484:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203686.748486:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748488:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.748490:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748493:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203686.748495:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203686.748499:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748503:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203686.748505:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203686.748512:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203686.748516:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748520:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203686.748524:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203686.748528:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748531:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203686.748535:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788203686.748537:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203686.748539:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203686.748541:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748554:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203686.748555:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203686.748557:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203686.748563:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203686.748569:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203686.748576:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543334272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203686.748585:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203686.748588:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203686.748592:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543334272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203686.748602:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203686.748605:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203686.748612:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203686.748613:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203686.748615:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203686.748620:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203686.748624:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25373 00000400:00000200:2.0:1788203686.748629:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203686.748630:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203686.748632:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203686.748637:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748639:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.748640:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748643:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203686.748648:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203686.748652:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203686.748655:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203686.748660:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203686.748665:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203686.748671:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203686.748680:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748685:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.748689:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748694:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203686.748697:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203686.748703:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748707:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203686.748709:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203686.748712:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203686.748715:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748724:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203686.748728:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203686.748732:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748734:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203686.748739:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788203686.748744:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203686.748748:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203686.748751:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748754:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203686.748757:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203686.748761:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203686.748765:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203686.748771:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203686.748780:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543334400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203686.748794:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203686.748799:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203686.748804:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543334400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203686.748809:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203686.748811:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203686.748816:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203686.748819:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203686.748822:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203686.748824:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203686.748827:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25373 00000400:00000200:2.0:1788203686.748833:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203686.748838:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203686.748841:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203686.748845:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748849:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.748852:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748853:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203686.748854:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203686.748855:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203686.748856:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203686.748857:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203686.748861:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203686.748863:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203686.748866:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:0.1:1788203686.748869:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203686.748874:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1788203686.748874:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000040:2.0:1788203686.748880:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748885:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203686.748889:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203686.748896:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203686.748899:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203686.748902:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203686.748907:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203686.748911:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748915:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203686.748918:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203686.748921:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748925:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203686.748932:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788203686.748935:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203686.748941:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203686.748944:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.748948:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203686.748951:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203686.748954:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203686.748960:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203686.748966:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203686.748973:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543334528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203686.748985:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203686.748989:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203686.748993:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543334528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203686.749002:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203686.749007:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203686.749010:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203686.749012:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203686.749015:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203686.749018:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203686.749020:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25373 00000400:00000200:2.0:1788203686.749025:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203686.749028:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203686.749029:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203686.749031:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.749033:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.749034:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.749035:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203686.749037:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203686.749038:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203686.749039:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203686.749050:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203686.749053:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203686.749056:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203686.749060:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.749063:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203686.749067:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203686.749069:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203686.749072:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203686.749076:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203686.749079:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203686.749083:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203686.749086:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203686.749089:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.749111:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203686.749112:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203686.749114:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.749115:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203686.749120:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203686.749122:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203686.749124:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203686.749125:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203686.749126:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203686.749127:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203686.749128:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203686.749131:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203686.749135:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203686.749143:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543334656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203686.749153:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203686.749159:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203686.749166:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543334656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203686.749175:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203686.749183:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25383) 00000800:00000200:0.1:1788203686.749461:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203686.749464:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203686.749684:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203686.749686:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203686.749845:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203686.749846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203686.750085:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203686.750087:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1788203687.580316:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203687.580321:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203687.580324:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203688.092156:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320046853, at 4320046912 00000100:00000010:2.0:1788203688.092179:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa5a0. 00000020:00000001:2.0:1788203688.604406:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203688.604411:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203688.604415:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203689.628384:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203689.628389:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203689.628393:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203690.652332:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203690.652337:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203690.652340:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203691.676303:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203691.676308:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203691.676312:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788203691.868830:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203691.868837:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203691.869448:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203691.869450:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203691.869662:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203691.869664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203691.869889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203691.869891:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203691.870117:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203691.870118:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1788203692.700375:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203692.700381:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203692.700384:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203693.212248:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320051973, at 4320052032 00000100:00000010:2.0:1788203693.212263:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcb40. 00000020:00000001:2.0:1788203693.724377:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203693.724383:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203693.724386:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203694.748332:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203694.748336:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203694.748339:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203695.772320:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203695.772325:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203695.772332:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203696.796339:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203696.796344:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203696.796348:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788203696.989010:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203696.989014:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203696.989686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203696.989688:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203696.989895:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203696.989897:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203696.990086:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203696.990106:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203696.990364:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203696.990379:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1788203697.820316:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203697.820321:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203697.820325:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203698.332185:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320057094, at 4320057152 00000100:00000010:2.0:1788203698.332204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc480. 00000020:00000001:2.0:1788203698.844407:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203698.844411:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203698.844413:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203699.868271:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203699.868273:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203699.868276:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203700.892371:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203700.892375:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203700.892378:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203701.916392:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203701.916398:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203701.916401:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203702.108245:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203702.108255:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203702.108258:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203702.108262:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203702.108266:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203702.108273:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25388 00000400:00000200:2.0:1788203702.108294:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203702.108301:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203702.108304:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203702.108306:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108309:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.108311:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108312:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203702.108316:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203702.108318:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203702.108320:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203702.108324:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203702.108326:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203702.108327:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203702.108329:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108330:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.108331:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108332:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203702.108334:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203702.108337:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108339:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203702.108342:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203702.108345:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203702.108347:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108349:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203702.108351:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203702.108361:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203702.108363:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203702.108367:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203702.108369:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108373:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203702.108376:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203702.108380:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203702.108391:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203702.108399:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203702.108410:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543336064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203702.108419:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203702.108424:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203702.108427:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543336064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203702.108449:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203702.108452:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203702.108455:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203702.108456:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203702.108457:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203702.108460:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203702.108462:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25388 00000400:00000200:2.0:1788203702.108465:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203702.108467:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203702.108468:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203702.108470:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108472:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.108473:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108475:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203702.108476:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203702.108477:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203702.108478:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203702.108480:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203702.108482:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203702.108483:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203702.108485:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108486:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.108487:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108489:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203702.108491:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203702.108493:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108495:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203702.108496:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203702.108501:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203702.108504:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108506:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203702.108511:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203702.108513:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108514:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203702.108516:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203702.108519:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203702.108523:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203702.108524:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108527:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203702.108528:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203702.108531:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203702.108536:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203702.108543:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203702.108553:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543336192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203702.108570:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203702.108574:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203702.108581:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543336192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203702.108594:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203702.108597:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203702.108600:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203702.108604:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203702.108606:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203702.108609:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203702.108615:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25388 00000400:00000200:2.0:1788203702.108620:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203702.108624:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203702.108626:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203702.108630:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108633:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.108638:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108640:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203702.108644:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203702.108647:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203702.108651:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203702.108657:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203702.108662:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203702.108667:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203702.108675:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108677:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.108680:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108684:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203702.108688:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203702.108693:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108695:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203702.108696:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203702.108699:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203702.108703:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108705:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203702.108706:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203702.108708:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108713:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203702.108719:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203702.108723:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203702.108729:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203702.108733:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108738:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203702.108742:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203702.108747:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203702.108755:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203702.108761:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203702.108777:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543336320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203702.108788:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203702.108792:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203702.108797:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543336320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203702.108804:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203702.108808:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203702.108813:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:3.1:1788203702.108814:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203702.108815:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203702.108818:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203702.108822:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.1:1788203702.108822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00080000:2.0:1788203702.108826:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25388 00000400:00000200:2.0:1788203702.108832:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203702.108836:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203702.108839:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203702.108843:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108845:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.108846:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108847:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203702.108848:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203702.108850:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203702.108853:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203702.108855:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203702.108858:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203702.108861:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203702.108867:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108869:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.108870:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108871:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203702.108876:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203702.108878:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203702.108882:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203702.108883:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203702.108885:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203702.108889:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108891:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203702.108893:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203702.108895:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108899:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203702.108904:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203702.108908:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203702.108911:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203702.108913:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.108917:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203702.108919:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203702.108921:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203702.108925:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203702.108929:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203702.108936:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543336448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203702.108944:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203702.108947:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203702.108952:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543336448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203702.108967:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203702.108969:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203702.108975:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203702.108977:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203702.108979:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203702.108982:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203702.108984:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25388 00000400:00000200:2.0:1788203702.108989:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203702.108992:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203702.108995:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203702.108997:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.109000:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.109003:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.109007:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203702.109010:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203702.109012:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203702.109015:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203702.109017:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203702.109019:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203702.109020:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203702.109021:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.109023:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203702.109026:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203702.109029:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203702.109031:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203702.109032:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203702.109033:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203702.109034:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203702.109037:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203702.109039:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.109042:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203702.109043:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203702.109045:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.109046:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203702.109049:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203702.109051:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203702.109053:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203702.109055:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203702.109057:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203702.109057:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203702.109058:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203702.109062:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203702.109066:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203702.109072:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543336576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203702.109078:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203702.109082:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203702.109085:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543336576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203702.109118:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203702.109121:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25398) 00000800:00000200:3.1:1788203702.109441:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203702.109443:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203702.109682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203702.109685:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203702.109868:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203702.109870:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788203702.110060:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203702.110062:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000020:00000001:2.0:1788203702.940337:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203702.940343:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203702.940347:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203703.452204:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320062213, at 4320062272 00000100:00000010:2.0:1788203703.452229:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc600. 00000020:00000001:2.0:1788203703.964366:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203703.964370:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203703.964372:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203704.988342:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203704.988346:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203704.988349:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788203706.012332:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203706.012336:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203706.012339:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788203707.036440:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203707.036446:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203707.036448:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203707.036451:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203707.036454:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203707.036457:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203707.036460:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203707.036463:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203707.036466:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203707.228271:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203707.228291:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203707.228294:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203707.228299:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203707.228304:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203707.228312:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25393 00000400:00000200:2.0:1788203707.228326:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203707.228334:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203707.228339:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203707.228343:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228346:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.228349:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228351:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203707.228356:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203707.228358:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203707.228361:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203707.228365:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203707.228368:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203707.228370:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203707.228373:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228376:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.228378:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228379:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203707.228382:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203707.228386:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228389:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203707.228393:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203707.228397:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203707.228400:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.228403:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203707.228406:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203707.228418:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788203707.228422:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203707.228427:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203707.228428:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.228432:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203707.228436:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203707.228442:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203707.228452:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203707.228460:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203707.228471:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543336704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203707.228480:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203707.228483:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203707.228487:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543336704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203707.228507:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203707.228515:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203707.228518:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203707.228519:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203707.228520:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203707.228522:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203707.228525:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25393 00000400:00000200:2.0:1788203707.228527:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203707.228530:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203707.228532:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203707.228534:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228536:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.228538:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228539:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203707.228540:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203707.228541:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203707.228542:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203707.228543:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203707.228545:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203707.228546:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203707.228548:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228551:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.228553:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228555:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203707.228557:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203707.228560:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228562:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203707.228564:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203707.228570:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203707.228572:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.228573:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203707.228577:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203707.228581:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.228583:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203707.228589:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788203707.228592:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203707.228596:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203707.228597:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.228599:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203707.228602:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203707.228605:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203707.228613:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203707.228617:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203707.228626:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543336832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203707.228686:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203707.228692:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203707.228704:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543336832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203707.228734:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203707.228741:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203707.228752:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203707.228755:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203707.228764:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203707.228770:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203707.228777:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25393 00000400:00000200:2.0:1788203707.228787:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203707.228795:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203707.228799:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203707.228806:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228821:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.228829:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228836:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203707.228842:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203707.228845:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203707.228850:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203707.228855:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203707.228860:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203707.228865:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203707.228874:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228880:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.228891:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228900:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203707.228905:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203707.228911:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203707.228917:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203707.228921:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203707.228930:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203707.228933:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.228936:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203707.228940:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203707.228951:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.228954:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203707.228961:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788203707.228969:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203707.228980:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203707.228986:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.228992:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203707.228996:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203707.229004:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203707.229016:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203707.229024:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203707.229041:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543336960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788203707.229063:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203707.229074:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203707.229077:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203707.229083:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543336960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203707.229126:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203707.229131:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203707.229135:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203707.229137:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203707.229140:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203707.229144:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203707.229148:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25393 00000400:00000200:2.0:1788203707.229152:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203707.229174:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203707.229182:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203707.229200:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229205:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.229209:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229213:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203707.229219:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203707.229220:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203707.229224:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203707.229227:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203707.229232:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203707.229233:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203707.229238:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229241:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.229244:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229249:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203707.229252:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203707.229256:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229263:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203707.229274:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203707.229281:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203707.229285:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.229288:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203707.229294:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203707.229303:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.229308:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203707.229318:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203707.229324:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203707.229328:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203707.229328:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.229330:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203707.229331:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203707.229333:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203707.229336:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203707.229341:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203707.229355:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543337088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203707.229382:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203707.229387:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203707.229393:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543337088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203707.229425:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203707.229432:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203707.229438:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203707.229443:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203707.229447:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203707.229456:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203707.229463:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25393 00000400:00000200:2.0:1788203707.229475:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203707.229481:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203707.229486:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203707.229495:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229498:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.229503:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229510:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203707.229517:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203707.229520:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203707.229526:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203707.229531:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203707.229534:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203707.229540:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203707.229556:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229562:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203707.229569:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229573:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203707.229578:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203707.229586:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203707.229592:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203707.229595:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203707.229601:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203707.229606:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.229614:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203707.229619:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203707.229627:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.229632:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203707.229641:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203707.229647:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203707.229655:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203707.229662:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203707.229672:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203707.229677:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203707.229682:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203707.229692:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203707.229703:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203707.229720:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543337216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203707.229753:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203707.229758:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203707.229772:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543337216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788203707.229789:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203707.229798:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203707.229808:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25403) 00000800:00000200:3.1:1788203707.229958:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203707.230229:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203707.230354:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203708.060192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203708.060201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203708.060205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203708.060210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203708.060217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203708.060227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203708.060234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203708.060238:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203708.060244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203708.572195:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320067334, at 4320067392 00000100:00000010:2.0:1788203708.572207:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc000. 00000100:00000001:2.0:1788203709.084193:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203709.084202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203709.084206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203709.084212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203709.084219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203709.084224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203709.084231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203709.084236:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203709.084242:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203709.084289:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203709.084293:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203709.084296:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788203710.108227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203710.108238:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203710.108243:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203710.108250:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203710.108260:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203710.108267:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203710.108278:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203710.108288:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203710.108298:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788203710.300458:0:256123:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788203710.300522:0:256123:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788203710.300535:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203710.300540:0:256123:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788203710.300547:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203710.300552:0:256123:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788203710.300559:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203710.300563:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd900. 00000400:00000200:2.0:1788203710.300570:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203710.300576:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203710.300585:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734dd0 00000400:00000010:2.0:1788203710.300588:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734dd0. 00000100:00000001:2.0:1788203710.300596:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203710.300607:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d76680 x1875055543336704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300644:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203710.300649:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76680 x1875055543336704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300658:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203710.300660:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203710.300675:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203710.300677:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:2.0:1788203710.300680:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203710.300683:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203710.300687:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734cc0 00000400:00000010:2.0:1788203710.300689:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734cc0. 00000100:00000001:2.0:1788203710.300692:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203710.300696:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75c00 x1875055543336832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300705:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203710.300708:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75c00 x1875055543336832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300714:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203710.300716:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203710.300720:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203710.300722:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd100. 00000400:00000200:2.0:1788203710.300726:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203710.300728:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203710.300732:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734880 00000400:00000010:2.0:1788203710.300733:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734880. 00000100:00000001:2.0:1788203710.300736:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203710.300741:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75180 x1875055543336960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300750:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203710.300754:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543336960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300760:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203710.300761:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203710.300767:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203710.300769:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd400. 00000400:00000200:2.0:1788203710.300771:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203710.300774:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203710.300776:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:2.0:1788203710.300778:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:2.0:1788203710.300780:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203710.300785:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75880 x1875055543337088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300791:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203710.300793:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543337088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300798:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203710.300800:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203710.300805:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203710.300807:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:2.0:1788203710.300809:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203710.300812:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203710.300814:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:2.0:1788203710.300817:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:2.0:1788203710.300819:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203710.300822:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74700 x1875055543337216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300828:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203710.300831:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543337216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203732 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203710.300837:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203710.300904:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788203711.132219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203711.132229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203711.132232:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203711.132237:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203711.132243:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203711.132248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203711.132253:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203711.132259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203711.132265:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203712.156217:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203712.156228:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203712.156231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203712.156235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203712.156240:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203712.156244:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203712.156248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203712.156252:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203712.156259:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203712.348904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203712.348911:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203712.349583:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203712.349586:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203712.349873:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203712.349875:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203712.350216:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203712.350219:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203712.350510:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203712.350512:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788203713.180207:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203713.180217:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203713.180222:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203713.180230:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203713.180237:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203713.180241:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203713.180244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203713.180249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203713.180253:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203713.692180:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320072454, at 4320072512 00000100:00000010:2.0:1788203713.692193:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc8a0. 00000100:00000001:2.0:1788203714.204157:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203714.204166:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203714.204169:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203714.204177:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203714.204185:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203714.204197:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203714.204201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203714.204205:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203714.204208:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203715.228181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203715.228190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203715.228193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203715.228200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203715.228207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203715.228213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203715.228219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203715.228232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203715.228236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203716.252144:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203716.252153:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203716.252156:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203716.252164:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203716.252176:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203716.252180:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203716.252185:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203716.252188:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203716.252192:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203717.276159:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203717.276165:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203717.276169:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203717.276175:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203717.276182:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203717.276187:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203717.276192:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203717.276195:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203717.276199:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203717.468696:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203717.468701:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203717.469272:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203717.469274:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203717.469465:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203717.469466:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203717.469647:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203717.469649:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203717.469828:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203717.469829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788203718.300180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203718.300189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203718.300194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203718.300202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203718.300210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203718.300217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203718.300224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203718.300237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203718.300242:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203718.812179:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320077573, at 4320077632 00000100:00000010:2.0:1788203718.812200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa900. 00000100:00000001:2.0:1788203719.324188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203719.324196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203719.324200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203719.324205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203719.324210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203719.324214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203719.324220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203719.324223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203719.324228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203720.348181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203720.348189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203720.348193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203720.348198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203720.348204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203720.348209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203720.348214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203720.348218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203720.348225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203721.372195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203721.372203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203721.372206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203721.372211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203721.372217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203721.372222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203721.372229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203721.372234:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203721.372238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203722.396189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203722.396198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203722.396202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203722.396208:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203722.396214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203722.396220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203722.396227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203722.396232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203722.396238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203722.588717:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203722.588723:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203722.589395:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203722.589397:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203722.589561:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203722.589563:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203722.589716:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203722.589717:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:0.1:1788203722.589951:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203722.589953:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1788203723.420180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203723.420187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203723.420198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203723.420205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203723.420208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203723.420211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203723.420216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203723.420219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203723.420224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203723.932192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320082693, at 4320082752 00000100:00000010:2.0:1788203723.932207:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa4e0. 00000100:00000001:2.0:1788203724.444182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203724.444190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203724.444207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203724.444214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203724.444220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203724.444223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203724.444228:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203724.444231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203724.444235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203725.468255:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203725.468267:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203725.468271:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203725.468282:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203725.468290:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203725.468301:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203725.468312:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203725.468315:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203725.468324:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203726.492210:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203726.492222:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203726.492228:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203726.492240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203726.492251:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203726.492257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203726.492268:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203726.492274:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203726.492279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.516198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203727.516207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203727.516210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203727.516214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203727.516221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203727.516226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203727.516230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203727.516235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203727.516241:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203727.708213:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203727.708226:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203727.708230:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203727.708234:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203727.708240:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203727.708249:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25414 00000400:00000200:2.0:1788203727.708262:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203727.708270:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203727.708274:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203727.708278:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708282:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.708285:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708288:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203727.708294:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203727.708296:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203727.708300:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203727.708304:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203727.708307:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203727.708310:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203727.708314:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708316:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.708317:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708320:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203727.708323:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203727.708326:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708328:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203727.708330:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203727.708334:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203727.708336:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708338:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203727.708340:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203727.708351:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203727.708354:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203727.708358:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203727.708359:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708362:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203727.708364:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203727.708369:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203727.708377:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203727.708384:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203727.708393:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543339264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203727.708402:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203727.708406:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203727.708410:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543339264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203727.708429:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203727.708432:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203727.708435:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203727.708436:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203727.708438:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203727.708440:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203727.708442:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25414 00000400:00000200:2.0:1788203727.708446:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203727.708448:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203727.708451:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203727.708453:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708455:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.708458:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708460:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203727.708461:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203727.708462:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203727.708464:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203727.708466:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203727.708468:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203727.708470:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203727.708480:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708483:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.708487:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708492:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203727.708498:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203727.708503:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708506:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203727.708509:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203727.708518:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203727.708522:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708525:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203727.708530:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203727.708536:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708539:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203727.708545:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203727.708547:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203727.708549:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203727.708552:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708554:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203727.708557:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203727.708561:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203727.708569:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203727.708577:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203727.708587:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543339392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203727.708602:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203727.708606:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203727.708612:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543339392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203727.708621:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203727.708626:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203727.708631:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203727.708635:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203727.708637:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203727.708639:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203727.708643:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25414 00000400:00000200:2.0:1788203727.708648:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203727.708654:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203727.708659:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203727.708665:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708667:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.708671:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708674:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203727.708676:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203727.708679:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203727.708681:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203727.708683:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203727.708688:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203727.708690:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203727.708694:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708697:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.708698:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708700:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203727.708704:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203727.708716:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708721:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203727.708725:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203727.708728:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203727.708730:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708731:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203727.708735:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203727.708740:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708742:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203727.708746:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203727.708750:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203727.708755:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203727.708761:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708767:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203727.708770:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203727.708773:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203727.708778:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203727.708782:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203727.708791:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543339520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203727.708800:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203727.708802:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203727.708809:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543339520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203727.708828:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203727.708846:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203727.708852:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203727.708853:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203727.708854:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203727.708858:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203727.708863:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25414 00000400:00000200:2.0:1788203727.708867:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203727.708872:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203727.708876:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203727.708881:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708883:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.708887:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708889:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203727.708892:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203727.708895:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000800:00000200:0.1:1788203727.708895:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788203727.708896:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203727.708900:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203727.708903:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:0.1:1788203727.708904:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000020:00000001:2.0:1788203727.708905:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203727.708911:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708914:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.708919:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708924:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203727.708926:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203727.708931:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203727.708933:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203727.708934:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203727.708937:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203727.708943:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708944:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203727.708948:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203727.708951:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708955:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203727.708958:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203727.708960:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203727.708965:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203727.708967:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.708971:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203727.708973:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203727.708975:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203727.708978:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203727.708983:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203727.708989:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543339648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203727.709013:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203727.709016:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203727.709026:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543339648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203727.709054:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203727.709060:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203727.709067:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203727.709071:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203727.709073:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203727.709078:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203727.709083:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25414 00000400:00000200:2.0:1788203727.709088:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203727.709121:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203727.709124:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203727.709129:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.709132:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.709135:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.709141:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203727.709143:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203727.709146:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203727.709151:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203727.709154:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203727.709158:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203727.709163:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203727.709167:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.709171:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203727.709173:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203727.709178:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203727.709180:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203727.709184:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203727.709189:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203727.709194:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203727.709199:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203727.709204:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.709209:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203727.709215:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203727.709220:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.709224:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203727.709230:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788203727.709235:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203727.709241:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203727.709243:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203727.709249:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203727.709253:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203727.709256:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203727.709261:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203727.709266:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203727.709275:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543339776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203727.709292:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203727.709296:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203727.709303:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543339776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203727.709335:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203727.709343:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25424) 00000800:00000200:0.1:1788203727.709569:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203727.709571:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203727.709766:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203727.709768:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203727.710006:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203727.710008:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:0.1:1788203727.710216:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203727.710217:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1788203728.540166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203728.540174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203728.540178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203728.540183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203728.540188:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203728.540202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203728.540210:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203728.540214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203728.540219:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203729.052181:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320087813, at 4320087872 00000100:00000010:2.0:1788203729.052194:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa7e0. 00000100:00000001:2.0:1788203729.564195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203729.564201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203729.564205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203729.564210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203729.564217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203729.564222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203729.564225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203729.564229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203729.564231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203730.588203:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203730.588214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203730.588229:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203730.588237:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203730.588244:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203730.588249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203730.588257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203730.588262:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203730.588266:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203731.612179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203731.612187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203731.612191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203731.612197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203731.612204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203731.612210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203731.612216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203731.612221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203731.612227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203732.636173:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203732.636182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203732.636187:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203732.636193:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203732.636200:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203732.636207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203732.636213:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203732.636217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203732.636221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203732.828741:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203732.828763:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203732.829457:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203732.829458:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203732.829648:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203732.829650:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203732.829848:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203732.829850:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203732.830116:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203732.830118:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1788203733.660157:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203733.660165:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203733.660167:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203733.660172:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203733.660176:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203733.660181:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203733.660186:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203733.660191:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203733.660194:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203734.172196:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320092933, at 4320092992 00000100:00000010:2.0:1788203734.172214:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faf00. 00000100:00000001:2.0:1788203734.684191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203734.684201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203734.684205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203734.684217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203734.684224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203734.684229:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203734.684236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203734.684243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203734.684251:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203735.708148:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203735.708155:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203735.708157:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203735.708162:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203735.708165:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203735.708168:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203735.708171:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203735.708174:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203735.708178:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203735.708249:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203735.708252:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203735.708254:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788203736.732186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203736.732195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203736.732200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203736.732207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203736.732214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203736.732219:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203736.732225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203736.732231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203736.732236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.756184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203737.756192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203737.756196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203737.756203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203737.756212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203737.756226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203737.756232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203737.756237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203737.756243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203737.948206:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203737.948214:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203737.948217:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203737.948221:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203737.948226:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203737.948234:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25424 00000400:00000200:2.0:1788203737.948244:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203737.948251:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203737.948254:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203737.948258:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948261:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.948264:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948267:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203737.948272:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203737.948274:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203737.948277:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203737.948280:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203737.948283:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203737.948286:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203737.948289:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948291:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.948294:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948296:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203737.948299:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203737.948303:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948306:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203737.948309:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203737.948313:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203737.948316:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.948319:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203737.948322:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203737.948338:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788203737.948343:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203737.948347:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203737.948349:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.948352:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203737.948356:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203737.948361:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203737.948371:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203737.948378:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203737.948387:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543340544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203737.948398:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203737.948403:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203737.948408:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543340544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203737.948433:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203737.948436:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203737.948439:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203737.948440:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203737.948441:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203737.948443:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203737.948446:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25424 00000400:00000200:2.0:1788203737.948449:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203737.948452:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203737.948454:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203737.948456:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948458:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.948461:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948463:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203737.948464:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203737.948465:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203737.948467:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203737.948469:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203737.948472:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203737.948475:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203737.948481:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948488:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.948494:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948500:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203737.948505:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203737.948514:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948520:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203737.948526:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203737.948534:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203737.948540:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.948545:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203737.948551:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203737.948558:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.948562:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203737.948567:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788203737.948572:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203737.948575:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203737.948579:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.948581:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203737.948583:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203737.948586:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203737.948591:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203737.948596:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203737.948601:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543340672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203737.948613:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203737.948616:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203737.948622:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543340672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203737.948653:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203737.948687:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203737.948695:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203737.948700:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203737.948705:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203737.948709:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203737.948715:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25424 00000400:00000200:2.0:1788203737.948722:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203737.948730:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203737.948733:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203737.948737:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948742:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.948747:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948753:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203737.948756:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203737.948758:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203737.948761:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203737.948764:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203737.948768:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203737.948771:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203737.948775:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948778:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.948781:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948785:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203737.948788:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000800:00000200:0.1:1788203737.948792:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788203737.948794:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948799:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203737.948799:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 02000000:00000001:2.0:1788203737.948802:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203737.948807:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203737.948811:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.948813:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203737.948817:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203737.948820:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.948824:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203737.948829:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203737.948833:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203737.948836:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203737.948838:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.948839:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203737.948841:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203737.948844:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203737.948851:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203737.948858:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203737.948867:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543340800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203737.948880:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203737.948883:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203737.948890:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543340800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203737.948914:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203737.948918:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203737.948925:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203737.948928:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203737.948931:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203737.948938:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203737.948944:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25424 00000400:00000200:2.0:1788203737.948949:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203737.948952:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203737.948956:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203737.948963:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948966:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.948970:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.948974:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203737.948980:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203737.948984:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203737.948988:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203737.948991:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203737.948994:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203737.948999:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203737.949003:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.949008:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.949012:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.949016:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203737.949020:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203737.949023:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203737.949027:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203737.949030:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203737.949035:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203737.949040:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.949044:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203737.949047:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203737.949053:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.949057:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203737.949064:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788203737.949070:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203737.949076:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203737.949079:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.949082:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203737.949086:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203737.949137:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203737.949145:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203737.949153:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203737.949165:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543340928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203737.949187:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203737.949193:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203737.949205:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543340928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203737.949225:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203737.949230:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203737.949237:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203737.949240:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203737.949243:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203737.949249:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203737.949256:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25424 00000400:00000200:2.0:1788203737.949263:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203737.949270:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203737.949276:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203737.949283:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.949289:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.949296:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.949300:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203737.949306:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203737.949309:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203737.949313:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203737.949315:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203737.949320:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203737.949326:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203737.949333:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.949337:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203737.949343:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203737.949348:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203737.949354:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203737.949360:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203737.949365:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203737.949371:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203737.949376:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203737.949380:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.949384:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203737.949388:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203737.949393:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.949397:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203737.949403:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203737.949408:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203737.949413:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203737.949416:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203737.949421:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203737.949425:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203737.949429:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203737.949436:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203737.949444:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203737.949454:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543341056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203737.949483:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203737.949488:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203737.949498:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543341056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203737.949519:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203737.949526:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25434) 00000800:00000200:0.1:1788203737.949575:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203737.949578:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203737.949865:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203737.949867:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203737.950085:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203737.950087:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203737.950428:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203737.950430:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1788203738.780200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203738.780212:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203738.780216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203738.780222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203738.780228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203738.780234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203738.780238:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203738.780243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203738.780256:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203738.780336:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203738.780341:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203738.780345:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203739.292175:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320098053, at 4320098112 00000100:00000010:2.0:1788203739.292190:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faf00. 00000100:00000001:2.0:1788203739.804196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203739.804205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203739.804209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203739.804216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203739.804230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203739.804237:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203739.804244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203739.804249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203739.804254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203740.828189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203740.828196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203740.828199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203740.828206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203740.828211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203740.828215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203740.828222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203740.828225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203740.828230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203741.852174:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203741.852184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203741.852188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203741.852196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203741.852204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203741.852210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203741.852216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203741.852218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203741.852225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203742.876186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203742.876195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203742.876200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203742.876206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203742.876212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203742.876216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203742.876222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203742.876228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203742.876235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203743.068188:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203743.068197:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203743.068199:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203743.068202:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203743.068205:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203743.068211:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25429 00000400:00000200:2.0:1788203743.068220:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203743.068226:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203743.068228:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203743.068230:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068233:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.068234:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068236:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203743.068239:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203743.068241:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203743.068243:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203743.068246:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203743.068248:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203743.068249:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203743.068251:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068253:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.068254:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068256:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203743.068258:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203743.068260:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068262:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203743.068264:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203743.068266:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203743.068269:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.068270:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203743.068279:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203743.068290:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203743.068292:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203743.068296:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203743.068298:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.068300:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203743.068303:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203743.068306:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203743.068312:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203743.068317:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203743.068324:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543341184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203743.068332:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203743.068336:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203743.068339:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543341184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203743.068356:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203743.068358:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203743.068361:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203743.068362:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203743.068363:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203743.068365:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203743.068367:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25429 00000400:00000200:2.0:1788203743.068369:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203743.068371:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203743.068373:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203743.068375:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068376:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.068378:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068380:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203743.068381:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203743.068383:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203743.068384:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203743.068386:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203743.068388:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203743.068389:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203743.068391:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068393:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.068394:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068395:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203743.068397:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203743.068401:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068404:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203743.068407:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203743.068412:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203743.068414:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.068415:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203743.068418:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203743.068421:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.068423:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203743.068434:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203743.068442:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203743.068447:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203743.068448:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.068450:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203743.068452:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203743.068454:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203743.068458:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203743.068465:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203743.068480:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543341312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203743.068509:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203743.068518:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203743.068525:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543341312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203743.068552:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203743.068555:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203743.068557:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203743.068559:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203743.068564:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203743.068580:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203743.068583:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25429 00000400:00000200:2.0:1788203743.068586:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203743.068588:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203743.068590:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203743.068592:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068594:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.068595:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068599:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203743.068603:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203743.068614:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203743.068616:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203743.068618:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203743.068620:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203743.068622:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203743.068635:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068637:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.068639:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068640:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203743.068648:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203743.068653:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068660:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203743.068663:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203743.068666:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203743.068668:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.068669:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203743.068671:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203743.068681:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.068683:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203743.068686:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203743.068688:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203743.068691:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203743.068695:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.068698:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203743.068706:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203743.068714:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203743.068724:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203743.068746:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203743.068758:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543341440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203743.068771:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203743.068776:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203743.068785:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543341440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203743.068800:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203743.068805:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203743.068815:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203743.068821:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203743.068839:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203743.068847:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203743.068856:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25429 00000400:00000200:2.0:1788203743.068871:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203743.068891:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203743.068899:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203743.068912:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068923:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1788203743.068930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788203743.068934:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:0.1:1788203743.068936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788203743.068943:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203743.068955:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203743.068957:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203743.068959:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203743.068962:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203743.068967:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203743.068968:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203743.068971:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068977:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.068989:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.068997:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203743.069001:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203743.069010:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203743.069016:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203743.069018:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203743.069023:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203743.069032:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.069039:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203743.069046:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203743.069053:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.069063:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203743.069073:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788203743.069080:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203743.069125:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203743.069129:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.069132:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203743.069135:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203743.069138:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203743.069143:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203743.069149:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203743.069158:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543341568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203743.069175:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203743.069180:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203743.069194:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543341568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203743.069231:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203743.069238:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203743.069254:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203743.069263:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203743.069271:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203743.069285:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203743.069296:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25429 00000400:00000200:2.0:1788203743.069309:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203743.069323:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203743.069335:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203743.069347:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.069354:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.069363:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.069372:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203743.069386:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203743.069391:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203743.069399:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203743.069408:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203743.069417:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203743.069430:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203743.069444:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.069456:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203743.069468:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203743.069479:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203743.069489:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203743.069505:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203743.069516:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203743.069524:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203743.069537:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203743.069544:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.069549:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203743.069555:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203743.069562:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.069583:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203743.069594:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788203743.069601:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203743.069609:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203743.069614:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203743.069623:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203743.069626:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203743.069628:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203743.069637:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203743.069649:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203743.069664:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543341696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203743.069691:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203743.069697:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203743.069710:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543341696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203743.069743:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203743.069754:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25439) 00000800:00000200:0.1:1788203743.069836:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203743.069839:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203743.070070:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203743.070072:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203743.070325:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203743.070329:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203743.070682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203743.070685:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788203743.900172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203743.900183:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203743.900189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203743.900195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203743.900203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203743.900207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203743.900218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203743.900224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203743.900229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203744.412189:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320103174, at 4320103232 00000100:00000010:2.0:1788203744.412203:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faba0. 00000100:00000001:2.0:1788203744.924236:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203744.924247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203744.924252:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203744.924260:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203744.924267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203744.924274:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203744.924282:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203744.924288:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203744.924296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203745.948160:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203745.948169:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203745.948174:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203745.948178:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203745.948183:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203745.948188:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203745.948191:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203745.948196:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203745.948200:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203746.972186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203746.972193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203746.972196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203746.972205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203746.972212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203746.972218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203746.972224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203746.972229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203746.972245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203747.996184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203747.996196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203747.996201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203747.996209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203747.996219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203747.996225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203747.996231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203747.996238:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203747.996246:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203748.188851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203748.188858:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203748.189673:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203748.189675:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203748.189922:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203748.189924:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203748.190154:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203748.190156:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:0.1:1788203748.190499:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203748.190501:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788203749.020257:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203749.020269:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203749.020277:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203749.020286:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203749.020291:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203749.020299:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203749.020308:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203749.020312:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203749.020316:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203749.532192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320108293, at 4320108352 00000100:00000010:2.0:1788203749.532214:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa4e0. 00000100:00000001:2.0:1788203750.044219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203750.044229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203750.044231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203750.044238:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203750.044247:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203750.044254:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203750.044270:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203750.044281:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203750.044290:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203751.068192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203751.068202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203751.068204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203751.068227:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203751.068241:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203751.068251:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203751.068264:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203751.068270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203751.068275:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203752.092255:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203752.092269:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203752.092282:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203752.092306:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203752.092325:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203752.092346:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203752.092359:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203752.092363:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203752.092368:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203753.116172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203753.116179:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203753.116181:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203753.116185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203753.116188:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203753.116193:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203753.116197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203753.116200:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203753.116205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203753.309071:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203753.309082:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203753.309984:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203753.309987:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203753.310253:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203753.310256:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203753.310572:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203753.310574:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203753.310845:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203753.310847:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788203754.140206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203754.140213:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203754.140216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203754.140222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203754.140229:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203754.140233:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203754.140238:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203754.140243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203754.140255:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203754.652156:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320113414, at 4320113472 00000100:00000010:2.0:1788203754.652167:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa900. 00000100:00000001:2.0:1788203755.164181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203755.164190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203755.164194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203755.164200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203755.164206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203755.164212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203755.164218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203755.164222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203755.164229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203756.188194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203756.188203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203756.188208:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203756.188216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203756.188223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203756.188227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203756.188231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203756.188236:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203756.188244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203757.212211:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203757.212217:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203757.212220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203757.212225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203757.212230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203757.212234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203757.212240:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203757.212243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203757.212246:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203758.236185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203758.236193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203758.236198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203758.236207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203758.236213:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203758.236219:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203758.236226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203758.236231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203758.236235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203758.428795:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203758.428801:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203758.429474:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203758.429477:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203758.429717:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203758.429720:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203758.429936:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203758.429938:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203758.430161:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203758.430163:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788203759.260178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203759.260185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203759.260190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203759.260195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203759.260201:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203759.260207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203759.260214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203759.260220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203759.260229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203759.772193:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320118533, at 4320118592 00000100:00000010:2.0:1788203759.772209:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa060. 00000100:00000001:2.0:1788203760.284167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203760.284175:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203760.284177:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203760.284183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203760.284187:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203760.284191:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203760.284197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203760.284201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203760.284206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203761.308208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203761.308214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203761.308216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203761.308220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203761.308224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203761.308227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203761.308230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203761.308232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203761.308235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203762.332191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203762.332199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203762.332205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203762.332214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203762.332221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203762.332225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203762.332230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203762.332249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203762.332257:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.356173:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203763.356180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203763.356185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203763.356192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203763.356198:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203763.356203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203763.356211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203763.356229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203763.356236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203763.548242:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203763.548253:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203763.548256:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203763.548260:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203763.548264:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203763.548270:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25449 00000400:00000200:2.0:1788203763.548278:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203763.548284:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203763.548286:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203763.548289:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548291:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.548293:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548295:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203763.548298:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203763.548300:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203763.548302:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203763.548305:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203763.548306:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203763.548308:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203763.548310:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548311:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.548313:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548314:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203763.548316:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203763.548318:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548320:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203763.548322:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203763.548324:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203763.548327:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.548329:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203763.548332:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203763.548339:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788203763.548342:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203763.548346:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203763.548347:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.548350:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203763.548352:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203763.548356:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203763.548361:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203763.548367:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203763.548374:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543343744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203763.548382:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203763.548386:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203763.548389:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543343744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203763.548408:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203763.548411:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203763.548414:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203763.548414:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203763.548416:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203763.548418:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203763.548420:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25449 00000400:00000200:2.0:1788203763.548423:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203763.548426:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203763.548429:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203763.548432:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548435:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.548438:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548441:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203763.548444:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203763.548445:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203763.548448:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203763.548452:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203763.548456:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203763.548461:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203763.548469:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548471:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.548476:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548481:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203763.548486:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203763.548491:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548494:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203763.548497:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203763.548501:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203763.548503:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.548505:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203763.548510:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203763.548515:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.548518:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203763.548523:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788203763.548528:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203763.548533:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203763.548535:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.548538:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203763.548540:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203763.548545:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203763.548553:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203763.548557:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203763.548563:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543343872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203763.548587:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203763.548593:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203763.548599:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543343872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203763.548617:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203763.548622:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203763.548632:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203763.548637:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203763.548640:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203763.548647:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203763.548653:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25449 00000400:00000200:2.0:1788203763.548660:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203763.548669:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203763.548676:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203763.548683:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548689:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.548695:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548701:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203763.548706:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203763.548709:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203763.548713:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203763.548717:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203763.548720:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203763.548725:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203763.548729:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548734:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.548738:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548741:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203763.548745:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203763.548750:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548754:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203763.548756:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203763.548758:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203763.548760:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.548763:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203763.548766:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203763.548778:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.548781:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203763.548786:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788203763.548791:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203763.548795:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203763.548798:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.548803:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:0.1:1788203763.548803:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:2.0:1788203763.548807:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.1:1788203763.548808:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 02000000:00000001:2.0:1788203763.548812:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203763.548820:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203763.548826:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203763.548838:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543344000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203763.548853:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203763.548858:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203763.548871:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543344000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203763.548900:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203763.548905:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203763.548909:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203763.548913:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203763.548916:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203763.548920:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203763.548926:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25449 00000400:00000200:2.0:1788203763.548931:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203763.548935:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203763.548938:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203763.548941:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548946:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.548949:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548952:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203763.548957:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203763.548958:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203763.548963:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203763.548966:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203763.548969:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203763.548972:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203763.548977:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548981:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.548984:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.548988:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203763.548993:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203763.548998:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203763.549002:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203763.549004:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203763.549007:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203763.549012:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.549016:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203763.549018:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203763.549022:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.549024:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203763.549029:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788203763.549035:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203763.549039:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203763.549041:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.549043:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203763.549044:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203763.549045:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203763.549048:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203763.549052:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203763.549058:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543344128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203763.549076:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203763.549079:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203763.549087:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543344128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203763.549122:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203763.549125:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203763.549130:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203763.549134:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203763.549138:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203763.549144:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203763.549151:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25449 00000400:00000200:2.0:1788203763.549161:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203763.549171:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203763.549177:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203763.549186:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.549193:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.549201:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.549206:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203763.549225:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203763.549228:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203763.549231:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203763.549234:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203763.549238:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203763.549244:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203763.549251:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.549254:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203763.549258:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203763.549263:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203763.549266:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203763.549272:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203763.549278:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203763.549282:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203763.549287:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203763.549291:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.549294:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203763.549297:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203763.549302:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.549305:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203763.549311:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788203763.549316:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203763.549320:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203763.549323:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203763.549327:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203763.549331:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203763.549332:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203763.549337:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203763.549343:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203763.549349:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543344256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203763.549361:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203763.549365:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203763.549370:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543344256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203763.549379:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203763.549386:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25460) 00000800:00000200:0.1:1788203763.549506:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203763.549508:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788203763.549735:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203763.549738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788203763.549933:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203763.549935:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:0.1:1788203763.550299:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203763.550302:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000100:00000001:2.0:1788203764.380184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203764.380192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203764.380208:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203764.380217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203764.380224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203764.380230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203764.380236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203764.380241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203764.380247:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203764.892169:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320123653, at 4320123712 00000100:00000010:2.0:1788203764.892183:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fae40. 00000100:00000001:2.0:1788203765.404172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203765.404191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203765.404194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203765.404199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203765.404205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203765.404210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203765.404214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203765.404219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203765.404225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203766.428230:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203766.428235:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203766.428237:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203766.428240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203766.428243:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203766.428245:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203766.428247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203766.428249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203766.428250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203767.452195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203767.452205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203767.452213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203767.452226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203767.452233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203767.452243:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203767.452253:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203767.452262:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203767.452269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203768.476243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203768.476260:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203768.476264:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203768.476271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203768.476277:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203768.476282:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203768.476289:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203768.476295:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203768.476303:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203768.668878:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203768.668883:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203768.669515:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203768.669517:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203768.669754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203768.669757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203768.669950:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203768.669952:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:0.1:1788203768.670207:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203768.670209:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1788203769.500221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203769.500235:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203769.500241:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203769.500260:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203769.500273:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203769.500279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203769.500285:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203769.500292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203769.500304:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203770.012232:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320128773, at 4320128832 00000100:00000010:2.0:1788203770.012245:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fab40. 00000100:00000001:2.0:1788203770.524206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203770.524214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203770.524219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203770.524224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203770.524230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203770.524235:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203770.524238:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203770.524242:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203770.524248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203771.548190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203771.548200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203771.548205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203771.548214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203771.548221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203771.548225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203771.548229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203771.548233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203771.548238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203772.572165:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203772.572170:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203772.572173:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203772.572178:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203772.572182:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203772.572186:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203772.572189:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203772.572192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203772.572195:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203773.596158:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203773.596165:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203773.596169:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203773.596177:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203773.596186:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203773.596200:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203773.596209:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203773.596215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203773.596220:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203773.788881:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203773.789263:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203773.789581:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203773.789778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203773.789916:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203774.620187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203774.620195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203774.620197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203774.620203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203774.620208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203774.620212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203774.620215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203774.620221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203774.620226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203775.132383:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320133893, at 4320133952 00000100:00000010:2.0:1788203775.132405:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc060. 00000100:00000001:2.0:1788203775.644316:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203775.644339:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203775.644342:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203775.644347:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203775.644353:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203775.644359:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203775.644363:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203775.644366:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203775.644369:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203776.668177:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203776.668184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203776.668189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203776.668201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203776.668206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203776.668210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203776.668215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203776.668218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203776.668222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203777.692224:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203777.692245:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203777.692249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203777.692253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203777.692258:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203777.692263:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203777.692268:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203777.692272:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203777.692282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203778.716191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203778.716199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203778.716202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203778.716211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203778.716219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203778.716225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203778.716231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203778.716237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203778.716245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203778.908939:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203778.908973:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203778.909765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203778.909768:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203778.910089:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203778.910116:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203778.910383:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203778.910385:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203778.910568:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203778.910569:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788203779.740201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203779.740210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203779.740215:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203779.740225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203779.740232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203779.740238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203779.740244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203779.740248:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203779.740252:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203780.252236:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320139014, at 4320139072 00000100:00000010:2.0:1788203780.252246:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa360. 00000100:00000001:2.0:1788203780.764202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203780.764211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203780.764216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203780.764225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203780.764238:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203780.764242:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203780.764247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203780.764251:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203780.764258:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203781.788198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203781.788208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203781.788214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203781.788223:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203781.788234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203781.788241:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203781.788249:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203781.788259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203781.788272:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203782.812212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203782.812224:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203782.812230:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203782.812239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203782.812250:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203782.812260:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203782.812267:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203782.812272:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203782.812280:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203783.836252:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203783.836263:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203783.836267:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203783.836282:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203783.836287:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203783.836297:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203783.836305:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203783.836313:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203783.836321:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203784.028252:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203784.028273:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203784.028276:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203784.028281:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203784.028286:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203784.028294:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25470 00000400:00000200:2.0:1788203784.028305:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203784.028313:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203784.028317:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203784.028320:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028322:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.028324:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028326:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203784.028330:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203784.028332:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203784.028335:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203784.028339:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203784.028340:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203784.028342:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203784.028344:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028346:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.028347:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028349:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203784.028351:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203784.028353:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028355:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203784.028358:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203784.028361:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203784.028364:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.028366:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203784.028369:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203784.028379:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203784.028382:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203784.028386:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203784.028387:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.028390:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203784.028393:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203784.028398:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203784.028406:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203784.028414:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203784.028424:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543346304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203784.028437:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203784.028451:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203784.028458:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543346304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203784.028484:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203784.028486:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203784.028490:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203784.028491:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203784.028492:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203784.028495:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203784.028497:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25470 00000400:00000200:2.0:1788203784.028500:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203784.028503:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203784.028504:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203784.028507:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028509:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.028510:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028513:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203784.028515:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203784.028516:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203784.028517:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203784.028519:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203784.028521:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203784.028522:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203784.028527:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028529:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.028532:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028533:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203784.028536:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203784.028539:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028541:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203784.028543:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203784.028550:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203784.028552:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.028554:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203784.028558:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203784.028562:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.028571:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203784.028578:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203784.028581:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203784.028587:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203784.028591:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.028600:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203784.028605:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203784.028609:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203784.028617:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203784.028626:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203784.028644:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543346432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203784.028670:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203784.028676:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203784.028685:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543346432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203784.028705:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203784.028711:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203784.028721:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203784.028723:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203784.028731:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203784.028736:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203784.028745:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25470 00000400:00000200:2.0:1788203784.028755:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203784.028761:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203784.028767:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203784.028774:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028779:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.028790:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028799:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203784.028821:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203784.028827:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203784.028834:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203784.028841:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203784.028847:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203784.028885:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203784.028896:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028900:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.028907:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028910:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203784.028913:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203784.028917:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203784.028921:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203784.028927:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203784.028931:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203784.028935:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.028940:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203784.028943:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203784.028949:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.028955:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203784.028963:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203784.028969:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203784.028979:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203784.028983:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.028990:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203784.028993:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203784.028998:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203784.029009:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203784.029016:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203784.029027:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543346560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1788203784.029064:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203784.029066:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:0.1:1788203784.029070:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788203784.029073:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203784.029138:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543346560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203784.029168:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203784.029172:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203784.029176:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203784.029177:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203784.029179:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203784.029182:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203784.029188:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25470 00000400:00000200:2.0:1788203784.029203:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203784.029216:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203784.029224:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203784.029240:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029245:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.029252:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029259:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203784.029264:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203784.029268:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203784.029273:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203784.029276:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203784.029282:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203784.029291:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203784.029299:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029305:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.029311:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029319:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203784.029322:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203784.029332:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029343:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203784.029346:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203784.029353:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203784.029358:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.029361:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203784.029365:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203784.029368:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.029376:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203784.029388:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203784.029396:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203784.029404:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203784.029408:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.029412:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203784.029414:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203784.029418:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203784.029428:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203784.029436:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203784.029451:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543346688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203784.029521:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203784.029526:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203784.029540:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543346688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203784.029570:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203784.029577:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203784.029594:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203784.029599:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203784.029603:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203784.029609:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203784.029616:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25470 00000400:00000200:2.0:1788203784.029628:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203784.029637:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203784.029642:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203784.029651:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029657:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.029663:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029671:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203784.029680:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203784.029684:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203784.029689:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203784.029695:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203784.029701:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203784.029708:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203784.029719:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029727:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203784.029739:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029753:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203784.029764:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203784.029780:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203784.029787:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203784.029794:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203784.029801:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203784.029808:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.029813:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203784.029818:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203784.029827:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.029830:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203784.029838:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788203784.029845:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203784.029852:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203784.029869:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203784.029872:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203784.029874:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203784.029880:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203784.029896:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788203784.029908:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203784.029931:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543346816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203784.029959:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203784.029967:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203784.029981:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543346816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203784.030019:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203784.030031:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25480) 00000800:00000200:0.1:1788203784.030134:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203784.030137:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203784.030349:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203784.030352:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203784.030557:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203784.030559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:0.1:1788203784.030877:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203784.030880:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788203784.860189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203784.860199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203784.860202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203784.860210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203784.860218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203784.860222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203784.860234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203784.860238:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203784.860243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203785.372223:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320144134, at 4320144192 00000100:00000010:2.0:1788203785.372335:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faa20. 00000100:00000001:2.0:1788203785.884195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203785.884205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203785.884209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203785.884214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203785.884219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203785.884223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203785.884230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203785.884233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203785.884238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203786.908182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203786.908191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203786.908193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203786.908198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203786.908203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203786.908217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203786.908221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203786.908225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203786.908230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203787.932167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203787.932175:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203787.932178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203787.932185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203787.932198:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203787.932202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203787.932209:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203787.932212:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203787.932217:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203788.956198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203788.956206:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203788.956210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203788.956217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203788.956223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203788.956228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203788.956233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203788.956237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203788.956244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203789.148617:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203789.148672:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203789.149133:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203789.149135:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203789.149331:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203789.149333:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203789.149521:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203789.149523:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203789.149701:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203789.149702:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788203789.980176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203789.980184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203789.980187:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203789.980196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203789.980203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203789.980208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203789.980211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203789.980216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203789.980224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203790.492173:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320149253, at 4320149312 00000100:00000010:2.0:1788203790.492194:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc1e0. 00000100:00000001:2.0:1788203791.004244:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203791.004252:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203791.004256:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203791.004264:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203791.004270:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203791.004276:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203791.004282:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203791.004287:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203791.004294:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203792.028310:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203792.028315:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203792.028316:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203792.028319:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203792.028323:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203792.028326:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203792.028329:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203792.028331:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203792.028333:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203793.052235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203793.052244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203793.052247:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203793.052261:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203793.052266:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203793.052272:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203793.052277:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203793.052280:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203793.052283:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203794.076163:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203794.076174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203794.076179:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203794.076190:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203794.076197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203794.076201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203794.076209:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203794.076216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203794.076220:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203794.268838:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203794.268846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203794.269645:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203794.269647:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203794.269873:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203794.269876:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203794.270085:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203794.270086:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203794.270297:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203794.270299:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1788203795.100200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203795.100209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203795.100212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203795.100217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203795.100223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203795.100228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203795.100233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203795.100237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203795.100240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203795.612159:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320154373, at 4320154432 00000100:00000010:2.0:1788203795.612168:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc5a0. 00000100:00000001:2.0:1788203796.124195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203796.124248:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203796.124251:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203796.124258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203796.124262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203796.124267:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203796.124271:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203796.124274:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203796.124278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203797.148188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203797.148194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203797.148198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203797.148203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203797.148209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203797.148214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203797.148219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203797.148224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203797.148231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203798.172161:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203798.172169:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203798.172173:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203798.172177:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203798.172183:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203798.172187:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203798.172190:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203798.172194:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203798.172201:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203799.196180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203799.196187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203799.196189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203799.196194:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203799.196198:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203799.196202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203799.196206:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203799.196210:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203799.196216:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203799.388737:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203799.388741:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203799.389339:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203799.389341:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203799.389551:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203799.389552:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203799.389768:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203799.389770:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788203799.390044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203799.390046:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1788203800.220200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203800.220209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203800.220213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203800.220220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203800.220226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203800.220230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203800.220237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203800.220242:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203800.220247:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203800.732201:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320159493, at 4320159552 00000100:00000010:2.0:1788203800.732215:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcde0. 00000100:00000001:2.0:1788203801.244192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203801.244201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203801.244204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203801.244210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203801.244213:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203801.244218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203801.244223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203801.244228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203801.244233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203802.268167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203802.268174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203802.268179:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203802.268186:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203802.268194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203802.268200:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203802.268207:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203802.268212:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203802.268218:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203803.292191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203803.292198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203803.292201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203803.292207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203803.292211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203803.292214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203803.292218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203803.292223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203803.292229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203804.316165:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203804.316172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203804.316174:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203804.316180:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203804.316183:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203804.316187:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203804.316193:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203804.316197:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203804.316200:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203804.508636:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203804.508644:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203804.510116:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203804.510119:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203804.510369:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203804.510372:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203804.510621:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203804.510622:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788203804.510820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203804.510823:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1788203805.340180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203805.340187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203805.340191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203805.340198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203805.340210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203805.340213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203805.340219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203805.340222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203805.340224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203805.852188:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320164614, at 4320164672 00000100:00000010:2.0:1788203805.852204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcba0. 00000100:00000001:2.0:1788203806.364159:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203806.364166:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203806.364170:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203806.364176:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203806.364182:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203806.364186:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203806.364190:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203806.364194:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203806.364198:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203807.388166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203807.388172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203807.388175:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203807.388183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203807.388190:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203807.388206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203807.388212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203807.388216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203807.388221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203808.412180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203808.412188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203808.412200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203808.412209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203808.412214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203808.412219:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203808.412223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203808.412227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203808.412230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203809.436176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203809.436184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203809.436187:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203809.436192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203809.436198:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203809.436203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203809.436208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203809.436211:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203809.436217:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203809.628732:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203809.628737:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203809.629175:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203809.629177:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203809.629366:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203809.629368:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203809.629566:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203809.629568:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203809.629814:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203809.629815:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788203810.460182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203810.460190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203810.460193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203810.460200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203810.460207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203810.460214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203810.460221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203810.460226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203810.460232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203810.972167:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320169733, at 4320169792 00000100:00000010:2.0:1788203810.972184:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fca20. 00000100:00000001:2.0:1788203811.484155:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203811.484162:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203811.484166:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203811.484173:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203811.484178:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203811.484183:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203811.484189:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203811.484192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203811.484196:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203812.508188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203812.508203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203812.508206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203812.508210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203812.508216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203812.508220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203812.508223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203812.508226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203812.508231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203813.532151:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203813.532160:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203813.532163:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203813.532169:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203813.532175:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203813.532178:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203813.532181:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203813.532184:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203813.532187:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203814.556186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203814.556205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203814.556209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203814.556214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203814.556218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203814.556223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203814.556227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203814.556231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203814.556235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203814.748708:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203814.748713:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203814.749288:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203814.749290:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203814.749681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203814.749683:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203814.749917:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203814.749920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203814.750105:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203814.750107:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788203815.580180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203815.580189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203815.580193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203815.580200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203815.580208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203815.580214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203815.580220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203815.580224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203815.580232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203816.092178:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320174853, at 4320174912 00000100:00000010:2.0:1788203816.092191:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fca80. 00000100:00000001:2.0:1788203816.604182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203816.604191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203816.604197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203816.604202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203816.604208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203816.604213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203816.604219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203816.604224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203816.604229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203817.628197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203817.628205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203817.628207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203817.628211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203817.628216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203817.628219:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203817.628222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203817.628227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203817.628232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203818.652181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203818.652189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203818.652192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203818.652195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203818.652201:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203818.652205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203818.652210:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203818.652214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203818.652221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203819.676152:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203819.676159:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203819.676161:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203819.676165:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203819.676168:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203819.676171:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203819.676173:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203819.676175:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203819.676177:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203819.868805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203819.868813:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203819.869454:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203819.869457:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203819.869724:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203819.869726:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203819.869969:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203819.869972:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203819.870273:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203819.870276:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788203820.700154:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203820.700161:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203820.700166:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203820.700173:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203820.700179:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203820.700184:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203820.700189:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203820.700192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203820.700195:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203821.212184:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320179973, at 4320180032 00000100:00000010:2.0:1788203821.212200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc420. 00000100:00000001:2.0:1788203821.724156:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203821.724163:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203821.724166:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203821.724173:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203821.724179:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203821.724184:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203821.724188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203821.724192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203821.724198:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203822.748196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203822.748205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203822.748207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203822.748214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203822.748221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203822.748225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203822.748231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203822.748235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203822.748240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203823.772182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203823.772189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203823.772193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203823.772204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203823.772210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203823.772215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203823.772221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203823.772225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203823.772231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203824.796196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203824.796204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203824.796208:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203824.796215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203824.796220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203824.796224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203824.796229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203824.796232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203824.796236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203824.988731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203824.988738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203824.989386:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203824.989387:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203824.989588:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203824.989590:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203824.989825:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203824.989827:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:0.1:1788203824.990028:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203824.990029:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788203825.820168:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203825.820176:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203825.820179:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203825.820185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203825.820188:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203825.820192:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203825.820196:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203825.820197:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203825.820200:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203826.332194:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320185093, at 4320185152 00000100:00000010:2.0:1788203826.332206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa540. 00000100:00000001:2.0:1788203826.844209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203826.844219:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203826.844222:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203826.844228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203826.844245:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203826.844250:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203826.844256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203826.844261:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203826.844266:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203827.868166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203827.868172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203827.868176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203827.868184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203827.868190:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203827.868192:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203827.868195:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203827.868198:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203827.868203:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203828.892168:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203828.892173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203828.892177:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203828.892181:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203828.892186:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203828.892191:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203828.892198:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203828.892201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203828.892206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203829.916164:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203829.916170:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203829.916174:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203829.916180:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203829.916185:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203829.916190:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203829.916197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203829.916203:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203829.916210:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203830.108743:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203830.108748:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203830.109242:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203830.109244:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203830.109479:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203830.109480:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203830.109680:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203830.109682:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:0.1:1788203830.109881:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203830.109883:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788203830.940170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203830.940178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203830.940182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203830.940187:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203830.940192:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203830.940196:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203830.940201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203830.940204:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203830.940209:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203831.452153:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320190213, at 4320190272 00000100:00000010:2.0:1788203831.452172:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa2a0. 00000100:00000001:2.0:1788203831.964174:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203831.964180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203831.964183:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203831.964189:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203831.964194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203831.964200:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203831.964204:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203831.964207:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203831.964211:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203832.988167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203832.988176:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203832.988180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203832.988185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203832.988191:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203832.988196:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203832.988201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203832.988206:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203832.988211:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203834.012178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203834.012187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203834.012192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203834.012201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203834.012208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203834.012212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203834.012216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203834.012221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203834.012227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203835.036143:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203835.036151:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203835.036153:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203835.036158:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203835.036163:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203835.036167:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203835.036170:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203835.036175:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203835.036180:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203835.228646:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203835.228650:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203835.229318:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203835.229322:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203835.229620:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203835.229622:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203835.229824:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203835.229826:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203835.230010:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203835.230012:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1788203836.060175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203836.060183:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203836.060186:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203836.060190:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203836.060197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203836.060202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203836.060208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203836.060213:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203836.060219:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203836.572160:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320195333, at 4320195392 00000100:00000010:2.0:1788203836.572173:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa3c0. 00000100:00000001:2.0:1788203837.084191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203837.084199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203837.084201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203837.084210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203837.084222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203837.084231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203837.084245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203837.084255:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203837.084260:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203838.108183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203838.108191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203838.108193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203838.108198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203838.108202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203838.108207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203838.108211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203838.108215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203838.108219:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203839.132183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203839.132199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203839.132206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203839.132213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203839.132219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203839.132224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203839.132229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203839.132233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203839.132237:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203840.156166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203840.156173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203840.156176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203840.156183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203840.156189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203840.156195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203840.156201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203840.156213:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203840.156221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203840.348910:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203840.349428:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203840.349944:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203840.350253:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203840.350394:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203841.180204:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203841.180215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203841.180220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203841.180228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203841.180235:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203841.180249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203841.180262:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203841.180271:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203841.180282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203841.692187:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320200453, at 4320200512 00000100:00000010:2.0:1788203841.692204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc9c0. 00000100:00000001:2.0:1788203842.204208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203842.204216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203842.204219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203842.204228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203842.204239:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203842.204249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203842.204253:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203842.204257:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203842.204269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203843.228164:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203843.228172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203843.228176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203843.228183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203843.228189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203843.228193:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203843.228196:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203843.228200:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203843.228205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788203843.420350:0:256120:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788203843.420395:0:256120:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788203843.420406:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203843.420410:0:256120:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788203843.420415:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203843.420420:0:256120:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788203843.420425:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203843.420428:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:2.0:1788203843.420434:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203843.420439:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203843.420445:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734d48 00000400:00000010:2.0:1788203843.420459:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734d48. 00000100:00000001:2.0:1788203843.420466:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203843.420475:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75da300 x1875055543353344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420500:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203843.420504:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da300 x1875055543353344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420512:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203843.420514:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203843.420525:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203843.420528:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdb00. 00000400:00000200:2.0:1788203843.420532:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203843.420536:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203843.420540:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:2.0:1788203843.420542:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:2.0:1788203843.420545:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203843.420551:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8e00 x1875055543353472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420563:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203843.420568:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8e00 x1875055543353472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420578:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203843.420582:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203843.420597:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203843.420604:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdd00. 00000400:00000200:2.0:1788203843.420610:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203843.420618:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203843.420624:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734088 00000400:00000010:2.0:1788203843.420628:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734088. 00000100:00000001:2.0:1788203843.420632:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203843.420642:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d9500 x1875055543353600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420661:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203843.420669:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9500 x1875055543353600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420682:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203843.420687:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203843.420703:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203843.420707:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:2.0:1788203843.420714:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203843.420722:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203843.420726:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:2.0:1788203843.420730:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:2.0:1788203843.420735:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203843.420742:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d9180 x1875055543353728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420756:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203843.420760:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9180 x1875055543353728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420770:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203843.420773:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203843.420782:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203843.420788:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:2.0:1788203843.420794:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203843.420799:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203843.420804:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734bb0 00000400:00000010:2.0:1788203843.420808:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734bb0. 00000100:00000001:2.0:1788203843.420813:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203843.420821:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75db800 x1875055543353856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420842:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203843.420846:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db800 x1875055543353856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203865 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203843.420852:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203843.420858:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788203844.252196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203844.252205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203844.252210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203844.252217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203844.252221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203844.252224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203844.252227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203844.252231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203844.252237:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203845.276154:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203845.276161:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203845.276165:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203845.276172:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203845.276178:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203845.276182:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203845.276188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203845.276200:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203845.276203:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203845.468820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203845.468827:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203845.469581:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203845.469587:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203845.469897:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203845.469900:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203845.470318:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203845.470320:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203845.470518:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203845.470520:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788203846.300166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203846.300174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203846.300177:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203846.300182:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203846.300189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203846.300195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203846.300202:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203846.300207:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203846.300212:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203846.812194:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320205573, at 4320205632 00000100:00000010:2.0:1788203846.812206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faea0. 00000100:00000001:2.0:1788203847.324178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203847.324186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203847.324190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203847.324196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203847.324201:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203847.324206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203847.324211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203847.324215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203847.324219:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203848.348188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203848.348196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203848.348198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203848.348202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203848.348208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203848.348211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203848.348215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203848.348218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203848.348221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203849.372185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203849.372192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203849.372196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203849.372202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203849.372208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203849.372212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203849.372218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203849.372221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203849.372224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203850.396197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203850.396216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203850.396220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203850.396225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203850.396228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203850.396232:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203850.396236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203850.396240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203850.396243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203850.588752:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203850.588766:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788203850.589394:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203850.589397:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788203850.589674:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203850.589677:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788203850.589962:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203850.589964:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:3.1:1788203850.590242:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203850.590245:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000100:00000001:2.0:1788203851.420196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203851.420213:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203851.420217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203851.420222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203851.420228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203851.420234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203851.420239:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203851.420242:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203851.420246:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203851.932229:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320210693, at 4320210752 00000100:00000010:2.0:1788203851.932237:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc420. 00000100:00000001:2.0:1788203852.444169:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203852.444176:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203852.444178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203852.444184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203852.444187:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203852.444191:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203852.444194:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203852.444196:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203852.444200:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203853.468176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203853.468185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203853.468190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203853.468197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203853.468204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203853.468217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203853.468223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203853.468226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203853.468229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203854.492178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203854.492186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203854.492189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203854.492196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203854.492202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203854.492208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203854.492215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203854.492221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203854.492227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203855.516185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203855.516192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203855.516194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203855.516201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203855.516206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203855.516211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203855.516217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203855.516223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203855.516232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203855.708638:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203855.708643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788203855.709208:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203855.709211:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788203855.709419:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203855.709421:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788203855.709579:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203855.709581:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788203855.709848:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203855.709850:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1788203856.540183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203856.540192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203856.540196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203856.540205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203856.540212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203856.540217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203856.540232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203856.540236:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203856.540241:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203857.052182:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320215813, at 4320215872 00000100:00000010:2.0:1788203857.052191:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fca80. 00000100:00000001:2.0:1788203857.564188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203857.564196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203857.564199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203857.564206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203857.564211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203857.564217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203857.564223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203857.564227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203857.564235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203858.588179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203858.588185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203858.588187:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203858.588191:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203858.588195:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203858.588201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203858.588206:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203858.588212:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203858.588218:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203859.612181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203859.612189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203859.612191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203859.612195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203859.612199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203859.612202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203859.612205:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203859.612210:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203859.612213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203860.636236:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203860.636244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203860.636246:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203860.636261:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203860.636270:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203860.636276:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203860.636282:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203860.636286:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203860.636289:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203860.828720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203860.828725:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203860.829345:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203860.829348:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203860.829651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203860.829653:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203860.829853:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203860.829855:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788203860.830055:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203860.830056:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1788203861.660183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203861.660190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203861.660193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203861.660199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203861.660204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203861.660209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203861.660215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203861.660218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203861.660222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203862.172193:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320220933, at 4320220992 00000100:00000010:2.0:1788203862.172206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fca20. 00000100:00000001:2.0:1788203862.684157:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203862.684166:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203862.684170:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203862.684176:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203862.684182:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203862.684185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203862.684188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203862.684192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203862.684198:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203863.708175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203863.708182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203863.708186:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203863.708192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203863.708196:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203863.708200:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203863.708205:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203863.708209:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203863.708214:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203864.732164:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203864.732171:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203864.732174:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203864.732179:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203864.732184:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203864.732189:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203864.732194:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203864.732198:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203864.732202:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203865.756185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203865.756193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203865.756196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203865.756203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203865.756208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203865.756213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203865.756217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203865.756221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203865.756226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203865.948616:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203865.948621:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203865.949159:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203865.949160:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203865.949335:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203865.949336:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203865.949510:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203865.949512:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788203865.949749:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203865.949751:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1788203866.780155:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203866.780162:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203866.780166:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203866.780173:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203866.780179:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203866.780184:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203866.780190:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203866.780194:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203866.780198:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203867.292179:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320226053, at 4320226112 00000100:00000010:2.0:1788203867.292193:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcba0. 00000100:00000001:2.0:1788203867.804191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203867.804196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203867.804199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203867.804202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203867.804206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203867.804210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203867.804215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203867.804218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203867.804222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203868.828180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203868.828188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203868.828191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203868.828197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203868.828203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203868.828209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203868.828214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203868.828218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203868.828225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203869.852185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203869.852193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203869.852197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203869.852202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203869.852207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203869.852211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203869.852215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203869.852219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203869.852230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203870.876183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203870.876190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203870.876194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203870.876197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203870.876202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203870.876206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203870.876211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203870.876215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203870.876220:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203871.068844:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203871.068852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203871.069372:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203871.069374:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203871.069656:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203871.069658:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203871.069941:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203871.069943:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203871.070237:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203871.070240:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1788203871.900191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203871.900198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203871.900202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203871.900207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203871.900211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203871.900215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203871.900220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203871.900225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203871.900228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203872.412165:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320231173, at 4320231232 00000100:00000010:2.0:1788203872.412183:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa3c0. 00000100:00000001:2.0:1788203872.924225:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203872.924242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203872.924249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203872.924255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203872.924263:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203872.924268:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203872.924270:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203872.924273:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203872.924276:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203873.948156:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203873.948162:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203873.948166:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203873.948172:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203873.948178:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203873.948182:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203873.948186:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203873.948190:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203873.948195:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203874.972186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203874.972193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203874.972198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203874.972204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203874.972210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203874.972215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203874.972220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203874.972224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203874.972228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203875.996169:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203875.996177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203875.996180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203875.996185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203875.996190:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203875.996194:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203875.996198:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203875.996201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203875.996205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203876.188758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203876.188763:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203876.189215:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203876.189216:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203876.189407:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203876.189409:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203876.189578:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203876.189579:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788203876.189761:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203876.189763:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788203877.020182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203877.020191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203877.020195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203877.020202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203877.020209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203877.020214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203877.020219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203877.020223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203877.020228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203877.532178:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320236293, at 4320236352 00000100:00000010:2.0:1788203877.532186:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc5a0. 00000100:00000001:2.0:1788203878.044173:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203878.044182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203878.044185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203878.044198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203878.044205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203878.044212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203878.044218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203878.044220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203878.044222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203879.068179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203879.068186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203879.068188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203879.068195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203879.068202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203879.068217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203879.068224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203879.068230:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203879.068238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203880.092170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203880.092177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203880.092181:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203880.092187:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203880.092201:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203880.092205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203880.092209:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203880.092212:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203880.092217:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203881.116212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203881.116221:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203881.116225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203881.116231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203881.116236:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203881.116241:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203881.116248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203881.116252:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203881.116258:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203881.308682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203881.308687:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203881.309203:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203881.309205:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203881.309627:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203881.309629:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203881.309911:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203881.309913:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203881.310153:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203881.310155:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788203882.140223:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203882.140237:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203882.140244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203882.140262:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203882.140274:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203882.140282:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203882.140294:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203882.140301:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203882.140305:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203882.652157:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320241413, at 4320241472 00000100:00000010:2.0:1788203882.652173:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc1e0. 00000100:00000001:2.0:1788203883.164191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203883.164199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203883.164204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203883.164211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203883.164219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203883.164225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203883.164232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203883.164237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203883.164244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203884.188170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203884.188176:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203884.188180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203884.188187:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203884.188193:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203884.188199:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203884.188204:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203884.188210:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203884.188213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203885.212206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203885.212214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203885.212217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203885.212225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203885.212231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203885.212235:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203885.212239:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203885.212245:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203885.212249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203886.236180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203886.236187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203886.236189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203886.236196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203886.236203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203886.236208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203886.236213:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203886.236218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203886.236225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203886.428726:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203886.428732:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203886.429383:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203886.429386:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203886.429637:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203886.429639:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203886.429886:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203886.429888:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:0.1:1788203886.430175:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203886.430177:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788203887.260174:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203887.260182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203887.260184:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203887.260189:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203887.260194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203887.260196:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203887.260200:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203887.260204:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203887.260211:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203887.772202:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320246533, at 4320246592 00000100:00000010:2.0:1788203887.772214:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faa80. 00000100:00000001:2.0:1788203888.284170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203888.284178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203888.284182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203888.284186:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203888.284192:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203888.284197:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203888.284202:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203888.284206:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203888.284211:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203889.308170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203889.308178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203889.308182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203889.308190:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203889.308196:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203889.308201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203889.308207:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203889.308212:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203889.308219:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203890.332151:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203890.332159:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203890.332162:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203890.332170:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203890.332175:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203890.332179:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203890.332182:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203890.332186:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203890.332189:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203891.356166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203891.356174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203891.356178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203891.356188:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203891.356210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203891.356216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203891.356221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203891.356224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203891.356228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1788203891.548715:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203891.548721:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788203891.549481:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203891.549483:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788203891.549678:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203891.549680:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788203891.549898:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203891.549900:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:0.1:1788203891.550240:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203891.550242:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000100:00000001:2.0:1788203892.380194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203892.380209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203892.380213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203892.380216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203892.380224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203892.380231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203892.380237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203892.380243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203892.380248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203892.892197:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320251653, at 4320251712 00000100:00000010:2.0:1788203892.892206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faa20. 00000100:00000001:2.0:1788203893.404192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203893.404202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203893.404205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203893.404211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203893.404215:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203893.404220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203893.404224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203893.404228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203893.404234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203894.428185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203894.428193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203894.428197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203894.428209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203894.428214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203894.428217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203894.428223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203894.428226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203894.428229:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203895.452214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203895.452227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203895.452236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203895.452242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203895.452250:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203895.452255:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203895.452265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203895.452271:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203895.452277:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203896.476191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203896.476199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203896.476203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203896.476209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203896.476214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203896.476217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203896.476222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203896.476226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203896.476231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203896.668600:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203896.668604:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203896.669240:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203896.669242:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203896.669459:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203896.669461:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203896.669646:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203896.669647:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788203896.669825:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203896.669827:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000100:00000001:2.0:1788203897.500194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203897.500203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203897.500209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203897.500216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203897.500225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203897.500231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203897.500236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203897.500240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203897.500244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203897.500282:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203897.500285:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203897.500287:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203898.012151:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320256773, at 4320256832 00000100:00000010:2.0:1788203898.012171:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc060. 00000100:00000001:2.0:1788203898.524171:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203898.524180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203898.524183:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203898.524191:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203898.524197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203898.524203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203898.524208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203898.524213:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203898.524218:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203899.548187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203899.548195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203899.548198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203899.548205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203899.548211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203899.548218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203899.548225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203899.548231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203899.548236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203900.572194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203900.572203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203900.572210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203900.572218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203900.572225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203900.572230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203900.572235:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203900.572241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203900.572249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203901.596192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203901.596202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203901.596208:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203901.596214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203901.596221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203901.596226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203901.596234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203901.596239:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203901.596244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203901.788859:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203901.788867:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203901.789427:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203901.789429:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203901.789653:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203901.789655:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203901.789859:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203901.789861:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788203901.790072:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203901.790073:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1788203902.620221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203902.620233:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203902.620238:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203902.620246:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203902.620254:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203902.620261:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203902.620264:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203902.620268:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203902.620272:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203903.132217:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320261893, at 4320261952 00000100:00000010:2.0:1788203903.132238:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc780. 00000100:00000001:2.0:1788203903.644214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203903.644226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203903.644232:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203903.644240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203903.644246:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203903.644259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203903.644272:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203903.644277:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203903.644282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203904.668226:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203904.668236:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203904.668244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203904.668261:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203904.668279:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203904.668290:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203904.668307:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203904.668311:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203904.668316:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203905.692162:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203905.692170:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203905.692173:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203905.692181:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203905.692189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203905.692195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203905.692201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203905.692205:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203905.692212:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203906.716232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203906.716243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203906.716248:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203906.716256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203906.716263:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203906.716269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203906.716274:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203906.716280:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203906.716286:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203906.909025:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203906.909413:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203906.909569:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203906.909751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203906.909935:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788203907.740219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203907.740232:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203907.740236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203907.740250:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203907.740256:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203907.740260:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203907.740262:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203907.740265:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203907.740269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203908.252224:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320267013, at 4320267072 00000100:00000010:2.0:1788203908.252243:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc000. 00000100:00000001:2.0:1788203908.764191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203908.764198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203908.764201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203908.764214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203908.764218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203908.764222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203908.764225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203908.764228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203908.764231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203908.764280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203908.764282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203909.788189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203909.788193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203909.788197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203909.788197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203909.788200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203909.788205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203909.788210:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788203909.788211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203909.788215:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788203909.788217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203909.788218:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203909.788222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203909.788223:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788203909.788226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203909.788227:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203909.788232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788238:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203909.788244:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203909.788246:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203909.788247:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788249:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203909.788250:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788253:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203909.788257:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203909.788258:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203909.788259:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788260:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203909.788260:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788261:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203909.788263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203910.812180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203910.812182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203910.812187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203910.812189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203910.812192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203910.812194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203910.812197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203910.812199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203910.812201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203910.812205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203910.812206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203910.812209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203910.812212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203910.812212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203910.812218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203910.812222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203910.812228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203911.836186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203911.836188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203911.836193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203911.836194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203911.836199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203911.836201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203911.836203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203911.836204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203911.836209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203911.836210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203911.836213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203911.836215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203911.836219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203911.836219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203911.836222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203911.836223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203911.836226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.028429:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203912.028436:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203912.028442:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203912.028453:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.028474:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203912.028480:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.028486:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203912.028492:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203912.028498:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203912.028506:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543362304:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203912.028517:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203912.028525:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203912.028530:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.028535:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203912.028544:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203912.028549:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203912.028561:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203912.028568:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203912.028580:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203912.028590:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543362304, portal 25 00000100:00000001:0.0:1788203912.028597:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203912.028601:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:0.0:1788203912.028613:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.028630:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203912.028669:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203912.028675:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203912.028682:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543362304, offset 0 00000400:00000200:0.0:1788203912.028697:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203912.028710:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.028722:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203912.028731:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203912.028739:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203912.028747:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203912.028755:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203912.028767:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470179:470179:256:4294967295] 192.168.201.118@tcp LPNI seq info [470038:470038:8:4294967295] 00000400:00000200:0.0:1788203912.028785:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.028802:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203912.028811:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.028821:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203912.028834:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203912.028841:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203912.028854:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203912.028862:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788203912.028871:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.028877:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203912.028885:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203912.028891:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203912.028901:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203912.028906:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203912.028915:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203912.028924:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.028944:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.028950:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.028958:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.028967:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203912.028974:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203912.028979:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203912.028983:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320272133 00000100:00000010:0.0:1788203912.028990:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa360. 00000100:00000001:0.0:1788203912.028998:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788203912.029006:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320272133 00000100:00000010:0.0:1788203912.029011:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fade0. 00000100:00000001:0.0:1788203912.029017:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788203912.029025:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203912.029031:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.029039:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203912.029050:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203912/real 1788203912] req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203912.029073:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203912.029076:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203912.029082:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203912.029133:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203912.029139:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203912.029146:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203912.029155:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203912.029182:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.029199:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203912.029203:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.029209:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.029214:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203912.029222:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.029252:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203912.029261:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.029277:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203912.029284:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.029289:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.029292:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.029300:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203912.029304:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203912.029312:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203912.029321:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203912.029328:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203912.029345:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203912.029366:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543362304:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203912.029376:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.029381:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.029389:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543362304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.029408:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203912.029411:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203912.029417:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203912.029422:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203912.029427:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203912.029432:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788203912.029437:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203912.029441:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:0.0:1788203912.029448:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203912.029452:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203912.029463:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:0.0:1788203912.029471:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203912.029476:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.029482:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.029497:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203912.029502:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.029509:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203912.029516:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203912.029520:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203912.029528:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.029551:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203912.029556:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.029562:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203912.029568:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203912.029573:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203912.029582:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543362432:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203912.029590:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203912.029594:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203912.029598:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.029601:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203912.029606:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203912.029610:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203912.029615:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203912.029618:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203912.029624:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203912.029628:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543362432, portal 10 00000100:00000001:0.0:1788203912.029631:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203912.029632:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:0.0:1788203912.029839:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.029847:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203912.029850:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203912.029854:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203912.029858:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543362432, offset 0 00000400:00000200:0.0:1788203912.029863:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203912.029869:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.029874:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203912.029878:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203912.029881:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203912.029885:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203912.029887:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203912.029891:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470180:470180:256:4294967295] 192.168.201.118@tcp LPNI seq info [470039:470039:8:4294967295] 00000400:00000200:0.0:1788203912.029896:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.029904:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203912.029910:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.029914:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203912.029920:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203912.029923:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203912.029928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203912.029931:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788203912.029935:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.029937:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203912.029941:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203912.029945:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203912.029948:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203912.029950:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203912.029953:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203912.029957:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.029966:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.029969:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.029976:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.029977:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203912.029981:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203912.029986:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788203912.029990:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203912.029992:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320270867 00000100:00000001:0.0:1788203912.029995:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320272133 00000100:00000001:0.0:1788203912.029998:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320270867 00000100:00000001:0.0:1788203912.030000:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320272133 00000100:00000001:0.0:1788203912.030003:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203912.030005:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.030007:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203912.030012:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203912/real 1788203912] req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203912.030020:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203912.030021:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203912.030023:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203912.030025:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203912.030027:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203912.030030:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203912.030033:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203912.030040:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030046:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203912.030048:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030050:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.030051:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203912.030055:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030062:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203912.030066:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030073:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203912.030077:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030078:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.030079:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030082:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203912.030085:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203912.030087:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203912.030117:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203912.030119:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203912.030124:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203912.030132:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543362432:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203912.030136:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203912.030140:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030146:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203912.030148:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.030150:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203912.030152:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203912.030155:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203912.030158:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543362560:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203912.030161:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203912.030163:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203912.030165:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.030167:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203912.030170:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788203912.030173:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203912.030176:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203912.030179:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203912.030182:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203912.030185:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543362560, portal 10 00000100:00000001:0.0:1788203912.030187:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203912.030189:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:0.0:1788203912.030195:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030202:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203912.030204:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203912.030206:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203912.030209:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543362560, offset 0 00000400:00000200:0.0:1788203912.030213:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203912.030217:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.030221:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203912.030224:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203912.030227:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203912.030230:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203912.030232:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203912.030235:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470181:470181:256:4294967295] 192.168.201.118@tcp LPNI seq info [470040:470040:8:4294967295] 00000400:00000200:0.0:1788203912.030239:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.030246:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203912.030251:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.030254:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203912.030258:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203912.030261:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203912.030264:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203912.030267:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788203912.030270:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.030273:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203912.030276:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203912.030279:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203912.030282:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203912.030284:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203912.030286:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203912.030291:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030299:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.030302:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030314:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.030316:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203912.030320:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203912.030323:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:0.0:1788203912.030326:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203912.030328:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320270869 00000100:00000001:0.0:1788203912.030331:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320272133 00000100:00000001:0.0:1788203912.030333:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320270869 00000100:00000001:0.0:1788203912.030335:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320272133 00000100:00000001:0.0:1788203912.030337:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203912.030339:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.030342:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203912.030346:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203912/real 1788203912] req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203912.030355:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203912.030357:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203912.030358:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203912.030361:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203912.030362:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203912.030365:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203912.030369:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203912.030377:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030384:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203912.030386:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030388:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.030389:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203912.030393:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030401:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203912.030405:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030412:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203912.030414:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030415:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.030417:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030419:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203912.030422:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203912.030424:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203912.030428:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203912.030430:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203912.030434:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203912.030443:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543362560:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203912.030447:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203912.030451:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030458:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203912.030459:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.030461:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203912.030463:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203912.030465:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203912.030468:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543362688:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203912.030472:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203912.030474:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203912.030476:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.030478:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203912.030481:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203912.030483:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203912.030487:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203912.030490:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203912.030494:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203912.030497:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543362688, portal 4 00000100:00000001:0.0:1788203912.030499:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203912.030500:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:0.0:1788203912.030507:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030513:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203912.030516:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203912.030518:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203912.030520:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543362688, offset 0 00000400:00000200:0.0:1788203912.030524:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203912.030528:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.030532:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203912.030536:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203912.030539:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203912.030541:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203912.030544:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203912.030547:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470182:470182:256:4294967295] 192.168.201.118@tcp LPNI seq info [470041:470041:8:4294967295] 00000400:00000200:0.0:1788203912.030551:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.030558:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203912.030562:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.030565:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203912.030570:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203912.030572:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203912.030577:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203912.030579:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788203912.030583:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.030585:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203912.030588:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203912.030592:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203912.030594:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203912.030596:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203912.030599:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203912.030603:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030611:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.030615:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030621:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.030623:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203912.030626:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203912.030630:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:0.0:1788203912.030633:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203912.030653:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320270878 00000100:00000001:0.0:1788203912.030655:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320272133 00000100:00000001:0.0:1788203912.030658:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320270878 00000100:00000001:0.0:1788203912.030659:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320272133 00000100:00000001:0.0:1788203912.030661:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203912.030662:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.030664:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203912.030667:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203912/real 1788203912] req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203912.030673:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203912.030674:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203912.030675:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203912.030677:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203912.030679:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203912.030681:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203912.030683:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203912.030688:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030692:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203912.030693:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030694:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.030695:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203912.030697:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030702:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203912.030705:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030709:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203912.030711:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030712:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.030712:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030714:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203912.030715:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203912.030716:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203912.030718:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203912.030720:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203912.030722:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203912.030727:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543362688:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203912.030729:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030731:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.030733:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543362432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030738:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203912.030740:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203912.030741:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203912.030743:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203912.030744:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203912.030745:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203912.030746:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203912.030748:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:0.0:1788203912.030750:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203912.030751:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203912.030752:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:0.0:1788203912.030754:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203912.030755:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030757:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.030759:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543362560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030763:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203912.030763:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203912.030764:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788203912.030766:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203912.030766:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203912.030767:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203912.030768:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203912.030769:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:0.0:1788203912.030770:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203912.030771:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203912.030772:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:0.0:1788203912.030773:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203912.030774:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030776:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.030778:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543362688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030782:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203912.030783:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203912.030784:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203912.030785:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203912.030786:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203912.030786:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203912.030788:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203912.030789:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:0.0:1788203912.030790:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203912.030790:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203912.030791:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:0.0:1788203912.030792:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203912.030793:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030794:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030798:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203912.030799:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.030801:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203912.030803:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203912.030804:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203912.030807:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030814:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203912.030816:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.030818:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203912.030820:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203912.030822:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203912.030825:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543362816:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203912.030829:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203912.030831:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203912.030833:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203912.030835:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203912.030838:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203912.030840:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203912.030844:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203912.030846:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203912.030850:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203912.030853:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543362816, portal 4 00000100:00000001:0.0:1788203912.030855:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203912.030856:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:0.0:1788203912.030862:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030866:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203912.030868:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203912.030870:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203912.030871:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543362816, offset 0 00000400:00000200:0.0:1788203912.030874:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203912.030877:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.030880:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203912.030882:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203912.030883:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203912.030885:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203912.030887:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203912.030889:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470183:470183:256:4294967295] 192.168.201.118@tcp LPNI seq info [470042:470042:8:4294967295] 00000400:00000200:0.0:1788203912.030892:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203912.030897:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203912.030900:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.030902:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203912.030905:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203912.030907:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:0.1:1788203912.030910:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203912.030912:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:0.0:1788203912.030914:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203912.030916:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203912.030918:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203912.030920:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203912.030922:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203912.030923:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203912.030925:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203912.030927:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030936:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.030938:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030943:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.030944:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203912.030945:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203912.030947:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:0.0:1788203912.030949:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203912.030950:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320270901 00000100:00000001:0.0:1788203912.030952:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320272133 00000100:00000001:0.0:1788203912.030954:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320270901 00000100:00000001:0.0:1788203912.030955:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320272133 00000100:00000001:0.0:1788203912.030957:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203912.030958:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.030959:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203912.030961:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203912/real 1788203912] req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203912.030966:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203912.030967:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203912.030968:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203912.030969:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203912.030970:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203912.030972:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203912.030974:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203912.030979:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030983:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203912.030984:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.030985:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.030986:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203912.030989:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030993:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203912.030995:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.030999:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203912.031000:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.031001:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203912.031002:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.031003:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203912.031004:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203912.031006:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203912.031008:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203912.031009:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203912.031011:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203912.031016:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543362816:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203912.031018:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.031019:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203912.031021:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543362816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203912.031026:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203912.031027:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203912.031028:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203912.031029:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203912.031030:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203912.031032:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203912.031033:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203912.031034:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:0.0:1788203912.031036:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203912.031036:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203912.031038:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:0.0:1788203912.031039:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203912.031040:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.031041:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203912.031043:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203912.031044:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203912.031045:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203912.031047:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203912.860162:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203912.860172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203912.860172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203912.860175:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203912.860181:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203912.860184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203912.860189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203912.860191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203912.860195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203912.860195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203912.860200:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203912.860201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203912.860204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203912.860205:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203912.860207:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203912.860208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203912.860209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203913.372176:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320272133, at 4320272192 00000100:00000010:2.0:1788203913.372192:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa360. 00000100:00000001:2.0:1788203913.884188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203913.884189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203913.884195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203913.884197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203913.884201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203913.884203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203913.884206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203913.884206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203913.884212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203913.884213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203913.884216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203913.884217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203913.884222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203913.884225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203913.884231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203913.884231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203913.884234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203914.908168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203914.908172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203914.908174:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203914.908178:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788203914.908178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203914.908182:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203914.908183:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203914.908188:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:2.0:1788203914.908189:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203914.908192:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203914.908197:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203914.908197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203914.908202:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203914.908202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203914.908206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203914.908207:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203914.908209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203914.908214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203914.908214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203914.908216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203914.908221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203914.908222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203914.908224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203915.932194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203915.932198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203915.932202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203915.932205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203915.932207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203915.932209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203915.932210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203915.932218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203915.932219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203915.932223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203915.932224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203915.932229:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203915.932231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203915.932236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203915.932237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203915.932243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203915.932249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203916.956176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203916.956185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203916.956185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203916.956189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203916.956191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203916.956193:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203916.956199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203916.956200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203916.956202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203916.956205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203916.956205:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203916.956209:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203916.956213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203916.956213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203916.956218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203916.956225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203916.956227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203917.148203:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203917.148213:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203917.148215:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203917.148218:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203917.148221:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203917.148227:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25603 00000400:00000200:0.0:1788203917.148235:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203917.148241:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203917.148243:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203917.148246:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148248:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.148250:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148252:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203917.148255:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203917.148257:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203917.148260:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203917.148263:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203917.148264:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203917.148266:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203917.148268:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148270:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.148271:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148272:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203917.148274:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203917.148276:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148278:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203917.148280:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203917.148283:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203917.148285:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148286:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203917.148288:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203917.148298:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788203917.148301:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203917.148305:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203917.148307:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148310:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203917.148313:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203917.148318:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203917.148328:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788203917.148335:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203917.148345:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543362944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203917.148357:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203917.148362:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203917.148366:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543362944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203917.148390:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203917.148392:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203917.148395:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203917.148396:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203917.148398:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203917.148400:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203917.148402:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25603 00000400:00000200:0.0:1788203917.148405:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203917.148408:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203917.148409:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203917.148411:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148413:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.148415:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148416:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203917.148418:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203917.148419:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203917.148421:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203917.148423:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203917.148426:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203917.148428:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203917.148432:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148434:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.148437:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148442:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203917.148445:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203917.148452:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148457:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203917.148462:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203917.148470:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203917.148475:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148480:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203917.148487:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203917.148494:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148500:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203917.148510:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788203917.148514:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203917.148519:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203917.148521:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148526:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203917.148527:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203917.148530:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203917.148536:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788203917.148541:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203917.148570:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543363072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203917.148581:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203917.148584:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203917.148589:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543363072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203917.148597:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203917.148600:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203917.148604:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203917.148608:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203917.148610:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203917.148614:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203917.148618:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25603 00000400:00000200:0.0:1788203917.148623:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203917.148628:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203917.148633:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203917.148640:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148645:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.148649:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148654:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203917.148659:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203917.148662:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203917.148666:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203917.148671:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203917.148676:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203917.148681:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203917.148688:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148690:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.148694:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148696:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203917.148698:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203917.148703:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148719:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203917.148727:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203917.148733:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203917.148738:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148744:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203917.148749:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203917.148754:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148760:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203917.148767:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788203917.148772:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203917.148779:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203917.148782:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148784:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203917.148786:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203917.148788:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203917.148791:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788203917.148796:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203917.148803:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543363200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203917.148825:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203917.148830:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203917.148839:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543363200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788203917.148855:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788203917.148856:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203917.148862:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00080000:0.0:1788203917.148870:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203917.148878:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788203917.148881:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203917.148883:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203917.148887:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203917.148893:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25603 00000400:00000200:0.0:1788203917.148903:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203917.148910:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203917.148915:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203917.148922:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148927:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.148931:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148933:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203917.148938:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203917.148940:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788203917.148942:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203917.148945:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203917.148946:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203917.148948:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203917.148950:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148951:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.148953:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148954:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203917.148958:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203917.148969:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203917.148973:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203917.148976:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203917.148979:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203917.148984:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148988:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203917.148991:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203917.148995:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.148999:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203917.149005:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788203917.149008:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203917.149013:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203917.149015:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.149020:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203917.149022:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203917.149025:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203917.149028:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788203917.149031:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203917.149040:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543363328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203917.149050:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203917.149055:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203917.149058:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543363328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203917.149068:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203917.149071:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203917.149078:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203917.149079:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203917.149081:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203917.149085:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203917.149112:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25603 00000400:00000200:0.0:1788203917.149117:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203917.149121:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203917.149123:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203917.149128:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.149130:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.149135:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.149137:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203917.149139:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203917.149142:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203917.149145:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203917.149147:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203917.149151:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203917.149153:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203917.149157:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.149159:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203917.149163:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203917.149164:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203917.149167:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203917.149171:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203917.149175:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203917.149178:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203917.149181:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203917.149185:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.149188:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203917.149190:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203917.149192:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.149196:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203917.149199:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788203917.149203:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203917.149207:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203917.149209:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203917.149213:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203917.149215:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203917.149217:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203917.149222:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788203917.149227:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203917.149233:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543363456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203917.149251:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203917.149254:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203917.149260:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543363456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203917.149268:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203917.149274:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25613) 00000800:00000200:3.1:1788203917.149585:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203917.149588:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203917.149907:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203917.149910:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203917.150237:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203917.150240:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203917.150572:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203917.150575:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788203917.980207:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203917.980211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203917.980217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203917.980219:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203917.980223:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203917.980227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203917.980231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203917.980232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203917.980240:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203917.980241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203917.980246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203917.980249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203917.980252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203917.980255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203917.980257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203917.980263:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203917.980269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203918.492176:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320277253, at 4320277312 00000100:00000010:2.0:1788203918.492192:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc600. 00000100:00000001:2.0:1788203919.004201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203919.004211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203919.004212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203919.004215:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203919.004217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203919.004224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203919.004225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203919.004229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203919.004230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203919.004233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203919.004235:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203919.004236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203919.004239:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203919.004242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203919.004243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203919.004244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203919.004250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203920.028200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203920.028202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203920.028207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203920.028211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203920.028214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203920.028216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028218:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203920.028223:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788203920.028224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203920.028226:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788203920.028232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203920.028234:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028238:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203920.028239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203920.028240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203920.028248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203920.028249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203920.028252:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203920.028254:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788203920.028256:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028257:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203920.028262:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028267:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203920.028269:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028271:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028273:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203920.028277:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203920.028278:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203920.028280:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028281:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203920.028283:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028284:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203920.028285:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203921.052156:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203921.052165:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203921.052168:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203921.052174:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203921.052176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203921.052180:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203921.052181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203921.052185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203921.052188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203921.052191:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203921.052192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203921.052197:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203921.052198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203921.052202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203921.052203:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203921.052206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203921.052209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203922.076203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203922.076210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203922.076217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203922.076222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203922.076231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203922.076235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203922.076241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203922.076244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203922.076319:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203922.076323:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203922.076324:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203922.076326:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203922.076327:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203922.076329:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203922.076330:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203922.076332:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203922.076334:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203922.268205:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203922.268214:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203922.268216:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203922.268220:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203922.268223:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203922.268229:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25608 00000400:00000200:0.0:1788203922.268237:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203922.268243:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203922.268245:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203922.268248:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268250:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.268252:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268255:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203922.268258:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203922.268260:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203922.268264:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203922.268268:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203922.268271:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203922.268273:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203922.268277:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268279:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.268281:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268283:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203922.268286:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203922.268290:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268293:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203922.268295:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203922.268299:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203922.268302:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.268305:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203922.268308:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203922.268318:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788203922.268321:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203922.268326:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203922.268328:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.268331:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203922.268334:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203922.268338:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203922.268346:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788203922.268352:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203922.268361:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543363584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203922.268372:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203922.268377:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203922.268382:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543363584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203922.268406:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203922.268410:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203922.268414:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203922.268416:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203922.268417:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203922.268421:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203922.268424:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25608 00000400:00000200:0.0:1788203922.268429:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203922.268439:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203922.268443:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203922.268449:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268455:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.268472:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268476:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203922.268482:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203922.268485:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203922.268490:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203922.268495:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203922.268501:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203922.268507:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203922.268517:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268521:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.268527:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268532:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203922.268538:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203922.268544:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268549:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203922.268554:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203922.268560:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203922.268565:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.268570:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203922.268577:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203922.268584:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.268589:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203922.268597:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788203922.268603:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203922.268611:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203922.268615:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.268619:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203922.268621:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203922.268626:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203922.268634:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788203922.268642:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203922.268653:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543363712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203922.268675:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203922.268681:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203922.268690:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543363712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203922.268707:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203922.268711:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203922.268716:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203922.268720:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203922.268722:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203922.268728:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.1:1788203922.268730:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788203922.268735:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25608 00000800:00000200:3.1:1788203922.268736:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000400:00000200:0.0:1788203922.268747:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203922.268755:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203922.268761:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203922.268769:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268774:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.268781:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268788:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203922.268794:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203922.268798:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203922.268802:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203922.268806:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203922.268811:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203922.268817:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203922.268824:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268829:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.268833:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268835:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203922.268839:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203922.268843:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268846:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203922.268849:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203922.268853:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203922.268855:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.268857:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203922.268859:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203922.268864:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.268865:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203922.268869:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788203922.268871:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203922.268876:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203922.268877:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.268879:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203922.268881:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203922.268884:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203922.268889:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788203922.268894:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203922.268901:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543363840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203922.268911:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203922.268916:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203922.268922:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543363840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203922.268932:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203922.268936:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203922.268942:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788203922.268946:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203922.268949:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203922.268954:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203922.268960:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25608 00000400:00000200:0.0:1788203922.268966:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203922.268973:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203922.268977:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203922.268982:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268986:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.268990:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.268996:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203922.269000:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203922.269003:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788203922.269006:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203922.269009:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203922.269012:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203922.269015:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203922.269020:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.269023:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.269028:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.269031:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203922.269034:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203922.269039:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203922.269043:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203922.269047:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203922.269050:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203922.269055:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.269058:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203922.269061:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203922.269065:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.269067:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203922.269070:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788203922.269074:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203922.269079:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203922.269081:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.269086:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203922.269088:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203922.269115:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203922.269119:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788203922.269125:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203922.269131:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543363968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203922.269144:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203922.269148:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203922.269156:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543363968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203922.269167:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203922.269172:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203922.269177:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203922.269181:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203922.269184:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203922.269187:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203922.269190:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25608 00000400:00000200:0.0:1788203922.269192:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203922.269194:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203922.269196:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203922.269202:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.269204:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.269208:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.269212:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203922.269215:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203922.269218:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203922.269221:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203922.269225:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203922.269227:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203922.269229:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203922.269232:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.269237:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203922.269242:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203922.269245:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203922.269246:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203922.269249:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203922.269250:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203922.269252:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203922.269254:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203922.269257:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.269261:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203922.269265:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203922.269273:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.269278:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203922.269286:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788203922.269293:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203922.269300:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203922.269305:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203922.269309:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203922.269314:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203922.269318:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203922.269328:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788203922.269334:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203922.269346:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543364096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203922.269362:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203922.269366:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203922.269373:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543364096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203922.269390:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203922.269398:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25618) 00000800:00000200:3.1:1788203922.269454:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203922.269470:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203922.269667:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203922.269669:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203922.269969:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203922.269972:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203922.270406:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203922.270409:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788203923.100177:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203923.100185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203923.100186:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203923.100191:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203923.100194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203923.100198:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203923.100200:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203923.100203:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203923.100206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203923.100213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203923.100221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203923.100229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203923.100231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203923.100236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203923.100237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203923.100242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203923.100243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203923.612255:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320282373, at 4320282432 00000100:00000010:2.0:1788203923.612290:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc480. 00000100:00000001:2.0:1788203924.124227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203924.124231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203924.124238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203924.124238:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203924.124242:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203924.124253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203924.124253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203924.124260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203924.124272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203924.124272:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203924.124280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203924.124285:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203924.124287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203924.124294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203924.124296:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203924.124300:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203924.124304:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203925.148152:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203925.148160:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203925.148162:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203925.148165:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203925.148167:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203925.148171:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788203925.148173:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203925.148174:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788203925.148177:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203925.148179:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203925.148181:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203925.148183:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788203925.148184:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203925.148187:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203925.148188:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203925.148190:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203925.148197:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203925.148201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203925.148203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203925.148209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203925.148212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203925.148221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203925.148225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203926.172183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203926.172185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203926.172190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203926.172191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203926.172196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203926.172198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203926.172202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203926.172202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203926.172208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203926.172210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203926.172212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203926.172213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203926.172218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203926.172220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203926.172222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203926.172224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203926.172226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203927.196194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203927.196196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203927.196201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203927.196203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203927.196207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203927.196208:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203927.196210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203927.196217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203927.196218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203927.196220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203927.196225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203927.196226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203927.196230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203927.196232:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203927.196239:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203927.196245:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203927.196254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203927.388188:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203927.388197:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203927.388199:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203927.388202:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203927.388205:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203927.388211:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25613 00000400:00000200:0.0:1788203927.388221:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203927.388228:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203927.388231:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203927.388235:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388238:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.388240:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388243:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203927.388247:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203927.388249:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203927.388252:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203927.388262:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203927.388265:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203927.388268:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203927.388271:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388274:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.388276:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388278:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203927.388281:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203927.388285:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388288:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203927.388292:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203927.388300:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203927.388304:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388306:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203927.388309:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203927.388318:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788203927.388322:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203927.388325:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203927.388326:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388329:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203927.388331:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203927.388334:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203927.388340:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788203927.388346:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203927.388354:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543364224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203927.388362:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203927.388366:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203927.388383:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543364224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203927.388404:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203927.388407:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203927.388409:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203927.388411:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203927.388412:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203927.388414:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203927.388416:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25613 00000400:00000200:0.0:1788203927.388420:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203927.388422:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203927.388424:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203927.388426:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388428:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.388430:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388431:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203927.388433:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203927.388434:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203927.388436:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203927.388438:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203927.388440:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203927.388441:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203927.388444:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388445:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.388448:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388451:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203927.388454:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203927.388459:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388463:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203927.388464:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203927.388471:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203927.388476:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388480:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203927.388484:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203927.388490:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388492:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203927.388496:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788203927.388502:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203927.388508:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203927.388511:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388516:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203927.388520:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203927.388526:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203927.388533:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788203927.388540:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203927.388548:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543364352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203927.388559:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203927.388561:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203927.388567:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543364352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203927.388581:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203927.388586:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203927.388595:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203927.388599:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203927.388602:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203927.388605:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203927.388612:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25613 00000400:00000200:0.0:1788203927.388619:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203927.388624:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203927.388628:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203927.388633:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388635:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.388637:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388640:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203927.388645:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203927.388647:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203927.388648:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203927.388652:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203927.388656:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203927.388660:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203927.388665:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388668:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.388674:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388678:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203927.388681:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203927.388688:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388694:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203927.388699:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203927.388703:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203927.388708:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388713:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203927.388716:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203927.388721:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388726:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203927.388730:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788203927.388733:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203927.388736:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203927.388739:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388744:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203927.388746:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203927.388749:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203927.388753:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000800:00000200:3.1:1788203927.388754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788203927.388759:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203927.388760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00100000:0.0:1788203927.388769:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543364480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203927.388781:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203927.388787:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203927.388796:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543364480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203927.388812:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203927.388819:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203927.388826:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788203927.388831:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203927.388836:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203927.388843:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203927.388850:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25613 00000400:00000200:0.0:1788203927.388861:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203927.388867:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203927.388869:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203927.388873:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388875:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.388878:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388881:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203927.388884:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203927.388886:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788203927.388890:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203927.388897:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203927.388904:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203927.388910:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203927.388921:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388926:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.388932:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388939:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203927.388946:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203927.388954:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203927.388959:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203927.388963:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203927.388968:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203927.388972:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388974:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203927.388975:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203927.388977:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388978:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203927.388983:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788203927.388987:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203927.388992:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203927.388994:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.388998:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203927.389001:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203927.389006:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203927.389010:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788203927.389016:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203927.389027:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543364608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203927.389041:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203927.389045:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203927.389054:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543364608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203927.389072:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203927.389077:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203927.389083:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203927.389085:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203927.389087:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203927.389121:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203927.389125:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25613 00000400:00000200:0.0:1788203927.389133:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203927.389142:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203927.389148:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203927.389156:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.389161:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.389168:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.389174:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203927.389180:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203927.389184:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203927.389189:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203927.389193:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203927.389198:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203927.389204:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203927.389214:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.389219:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203927.389225:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203927.389230:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203927.389232:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203927.389237:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203927.389241:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203927.389244:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203927.389248:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203927.389252:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.389253:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203927.389255:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203927.389261:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.389266:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203927.389273:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788203927.389277:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203927.389291:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203927.389294:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203927.389301:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203927.389304:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203927.389309:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203927.389315:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788203927.389321:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203927.389331:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543364736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203927.389342:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203927.389346:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203927.389351:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543364736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203927.389359:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203927.389364:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25623) 00000800:00000200:3.1:1788203927.389443:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203927.389445:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203927.389642:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203927.389645:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203927.389860:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203927.389862:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203927.390194:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203927.390196:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1788203928.220191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203928.220195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203928.220199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203928.220200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203928.220203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203928.220206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203928.220209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203928.220210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203928.220216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203928.220218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203928.220220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203928.220221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203928.220225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203928.220229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203928.220229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203928.220232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203928.220234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203928.732179:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320287493, at 4320287552 00000100:00000010:2.0:1788203928.732205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcb40. 00000100:00000001:2.0:1788203929.244199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203929.244205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203929.244207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203929.244210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203929.244211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203929.244218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203929.244222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203929.244224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203929.244227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203929.244227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203929.244232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203929.244236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203929.244237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203929.244240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203929.244243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203929.244247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203929.244250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203930.268176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203930.268183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203930.268184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203930.268189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203930.268197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203930.268199:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788203930.268202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203930.268204:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788203930.268207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203930.268210:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203930.268213:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203930.268216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788203930.268217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203930.268220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203930.268222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203930.268236:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203930.268238:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203930.268240:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203930.268244:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268246:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268248:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203930.268254:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203930.268255:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203930.268257:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268259:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203930.268260:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268262:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203930.268264:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203931.292182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203931.292185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203931.292190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203931.292191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203931.292195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203931.292198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203931.292202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203931.292203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203931.292204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203931.292208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203931.292211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203931.292212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203931.292215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203931.292217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203931.292222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203931.292226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203931.292233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203932.316166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203932.316171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203932.316173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203932.316176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203932.316177:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203932.316185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203932.316186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203932.316189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203932.316192:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203932.316198:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203932.316200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203932.316205:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203932.316207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203932.316211:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203932.316213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203932.316216:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203932.316217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.508432:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203932.508438:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203932.508442:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203932.508451:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.508462:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203932.508465:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.508469:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203932.508472:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203932.508476:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203932.508483:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543364864:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203932.508490:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203932.508494:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203932.508497:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.508500:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203932.508507:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203932.508511:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203932.508519:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203932.508525:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203932.508533:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203932.508541:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543364864, portal 25 00000100:00000001:0.0:1788203932.508545:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203932.508548:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:0.0:1788203932.508555:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.508563:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203932.508568:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203932.508572:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203932.508576:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543364864, offset 0 00000400:00000200:0.0:1788203932.508584:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203932.508594:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.508603:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203932.508609:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203932.508614:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203932.508619:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203932.508623:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203932.508628:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470199:470199:256:4294967295] 192.168.201.118@tcp LPNI seq info [470058:470058:8:4294967295] 00000400:00000200:0.0:1788203932.508633:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.508646:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203932.508657:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.508665:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203932.508676:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203932.508682:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203932.508691:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203932.508697:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788203932.508704:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.508708:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203932.508714:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203932.508723:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203932.508728:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203932.508732:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203932.508737:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203932.508744:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.508760:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.508768:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.508779:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.508782:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203932.508785:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203932.508793:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203932.508797:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320292613 00000100:00000010:0.0:1788203932.508805:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fab40. 00000100:00000001:0.0:1788203932.508811:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788203932.508818:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320292613 00000100:00000010:0.0:1788203932.508822:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fac00. 00000100:00000001:0.0:1788203932.508826:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788203932.508831:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203932.508837:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.508839:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203932.508842:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203932/real 1788203932] req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203932.508849:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203932.508850:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203932.508852:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203932.508858:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203932.508862:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203932.508865:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203932.508871:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203932.508890:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.508920:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203932.508924:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.508929:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.508932:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203932.508941:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.508952:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203932.508959:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.508972:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203932.508976:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.508981:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.508984:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.508988:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203932.508992:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203932.508997:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203932.509006:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203932.509010:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203932.509016:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203932.509022:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543364864:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203932.509028:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509032:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.509037:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543364864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509065:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203932.509069:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203932.509072:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203932.509077:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203932.509079:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203932.509082:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788203932.509086:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203932.509108:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:0.0:1788203932.509112:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203932.509114:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203932.509115:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:0.0:1788203932.509118:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203932.509121:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509127:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509137:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203932.509140:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.509143:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203932.509146:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203932.509148:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203932.509153:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509163:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203932.509166:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.509168:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203932.509171:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203932.509175:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203932.509182:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543364992:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203932.509187:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203932.509191:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203932.509194:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.509197:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203932.509200:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203932.509204:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203932.509211:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203932.509216:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203932.509222:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203932.509226:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543364992, portal 10 00000100:00000001:0.0:1788203932.509229:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203932.509232:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:0.0:1788203932.509242:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509256:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203932.509262:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203932.509267:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203932.509273:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543364992, offset 0 00000400:00000200:0.0:1788203932.509298:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203932.509305:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.509312:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203932.509318:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203932.509322:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203932.509328:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203932.509334:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203932.509340:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470200:470200:256:4294967295] 192.168.201.118@tcp LPNI seq info [470059:470059:8:4294967295] 00000400:00000200:0.0:1788203932.509347:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.509357:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203932.509366:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.509373:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203932.509381:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203932.509385:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203932.509391:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203932.509396:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788203932.509401:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.509403:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203932.509407:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203932.509415:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203932.509421:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203932.509425:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203932.509429:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203932.509435:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509446:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.509451:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509460:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.509462:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203932.509465:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203932.509468:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788203932.509470:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203932.509472:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320291347 00000100:00000001:0.0:1788203932.509475:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320292613 00000100:00000001:0.0:1788203932.509477:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320291347 00000100:00000001:0.0:1788203932.509478:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320292613 00000100:00000001:0.0:1788203932.509480:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203932.509481:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.509482:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203932.509485:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203932/real 1788203932] req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203932.509490:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203932.509491:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203932.509492:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203932.509494:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203932.509495:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203932.509497:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203932.509499:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203932.509504:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509512:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203932.509513:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509514:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.509515:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203932.509518:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509522:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203932.509524:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509528:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203932.509530:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509531:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.509531:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509533:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203932.509534:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203932.509536:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203932.509538:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203932.509539:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203932.509541:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203932.509546:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543364992:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203932.509548:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203932.509550:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509555:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203932.509556:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.509558:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203932.509559:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203932.509560:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203932.509562:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365120:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203932.509564:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203932.509566:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203932.509567:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.509568:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203932.509570:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203932.509571:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203932.509573:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203932.509575:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203932.509577:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203932.509579:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543365120, portal 10 00000100:00000001:0.0:1788203932.509580:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203932.509581:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:0.0:1788203932.509585:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509589:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203932.509591:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203932.509592:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203932.509594:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543365120, offset 0 00000400:00000200:0.0:1788203932.509596:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203932.509599:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.509601:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203932.509603:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203932.509605:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203932.509607:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203932.509608:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203932.509610:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470201:470201:256:4294967295] 192.168.201.118@tcp LPNI seq info [470060:470060:8:4294967295] 00000400:00000200:0.0:1788203932.509613:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.509617:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203932.509620:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.509622:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203932.509624:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203932.509626:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203932.509629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203932.509631:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788203932.509633:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.509634:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203932.509636:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203932.509638:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203932.509641:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203932.509642:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203932.509643:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203932.509646:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509651:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.509652:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509657:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.509658:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203932.509660:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203932.509661:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:0.0:1788203932.509663:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203932.509665:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320291348 00000100:00000001:0.0:1788203932.509666:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320292613 00000100:00000001:0.0:1788203932.509668:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320291348 00000100:00000001:0.0:1788203932.509669:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320292613 00000100:00000001:0.0:1788203932.509670:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203932.509671:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.509672:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203932.509675:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203932/real 1788203932] req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203932.509680:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203932.509681:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203932.509682:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203932.509684:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203932.509685:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203932.509686:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203932.509688:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203932.509693:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509697:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203932.509698:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509699:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.509700:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203932.509702:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509706:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203932.509708:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509712:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203932.509713:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509714:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.509715:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509716:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203932.509718:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203932.509719:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203932.509721:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203932.509722:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203932.509725:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203932.509730:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365120:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203932.509732:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509733:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.509735:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543364992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509739:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203932.509740:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203932.509741:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203932.509743:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203932.509743:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203932.509744:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203932.509746:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203932.509747:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:0.0:1788203932.509748:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203932.509749:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203932.509750:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:0.0:1788203932.509752:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203932.509753:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509754:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.509756:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543365120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509760:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203932.509760:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203932.509761:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203932.509762:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203932.509763:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203932.509764:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203932.509765:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203932.509766:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:0.0:1788203932.509767:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203932.509768:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203932.509769:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:0.0:1788203932.509771:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203932.509771:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509773:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509775:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203932.509776:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.509777:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203932.509778:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203932.509779:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203932.509782:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509786:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203932.509787:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.509789:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203932.509790:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203932.509791:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203932.509793:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365248:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203932.509795:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203932.509796:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203932.509797:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.509798:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203932.509800:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203932.509801:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203932.509803:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203932.509804:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203932.509806:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203932.509808:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543365248, portal 4 00000100:00000001:0.0:1788203932.509809:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203932.509810:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:0.0:1788203932.509813:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509817:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203932.509819:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203932.509820:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203932.509822:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543365248, offset 0 00000400:00000200:0.0:1788203932.509824:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203932.509826:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.509828:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203932.509830:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203932.509832:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203932.509834:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203932.509835:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203932.509837:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470202:470202:256:4294967295] 192.168.201.118@tcp LPNI seq info [470061:470061:8:4294967295] 00000400:00000200:0.0:1788203932.509840:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.509844:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203932.509847:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.509848:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203932.509851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203932.509852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203932.509855:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203932.509857:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788203932.509859:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.509860:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203932.509862:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203932.509864:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203932.509866:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203932.509867:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203932.509868:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203932.509871:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509879:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.509882:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509889:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.509891:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203932.509895:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203932.509898:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:0.0:1788203932.509901:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203932.509903:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320291357 00000100:00000001:0.0:1788203932.509906:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320292613 00000100:00000001:0.0:1788203932.509908:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320291357 00000100:00000001:0.0:1788203932.509910:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320292613 00000100:00000001:0.0:1788203932.509913:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203932.509915:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.509917:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203932.509921:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203932/real 1788203932] req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203932.509930:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203932.509931:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203932.509932:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203932.509934:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203932.509935:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203932.509936:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203932.509939:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203932.509944:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509948:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203932.509949:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509950:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.509951:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203932.509953:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509957:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203932.509959:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509964:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203932.509966:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509967:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.509968:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.509969:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203932.509970:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203932.509972:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203932.509974:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203932.509975:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203932.509977:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203932.509982:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365248:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203932.509985:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203932.509987:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.509991:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203932.509992:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.509993:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203932.509994:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203932.509995:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203932.509997:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365376:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203932.509999:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203932.510000:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203932.510001:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203932.510002:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203932.510004:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203932.510006:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203932.510008:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203932.510010:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203932.510012:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203932.510014:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543365376, portal 4 00000100:00000001:0.0:1788203932.510015:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203932.510016:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:0.0:1788203932.510020:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.510025:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203932.510026:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203932.510028:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203932.510029:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543365376, offset 0 00000400:00000200:0.0:1788203932.510032:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203932.510034:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.510036:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203932.510038:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203932.510040:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203932.510041:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203932.510043:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203932.510045:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470203:470203:256:4294967295] 192.168.201.118@tcp LPNI seq info [470062:470062:8:4294967295] 00000400:00000200:0.0:1788203932.510047:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203932.510052:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203932.510054:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.510056:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203932.510059:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203932.510060:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:0.1:1788203932.510063:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203932.510065:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:0.0:1788203932.510068:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203932.510069:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203932.510071:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203932.510073:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203932.510075:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203932.510077:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203932.510078:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203932.510080:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.510085:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.510087:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203957 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.510106:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.510108:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203932.510110:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203932.510112:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:0.0:1788203932.510114:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203932.510115:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320291381 00000100:00000001:0.0:1788203932.510117:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320292613 00000100:00000001:0.0:1788203932.510119:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320291381 00000100:00000001:0.0:1788203932.510120:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320292613 00000100:00000001:0.0:1788203932.510122:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203932.510123:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.510124:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203932.510126:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203932/real 1788203932] req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203932.510133:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203932.510135:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203932.510136:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203932.510138:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203932.510140:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203932.510142:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203932.510146:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203932.510155:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.510168:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203932.510170:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.510172:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.510174:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203932.510177:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.510185:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203932.510188:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.510193:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203932.510195:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.510196:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203932.510196:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.510198:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203932.510199:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203932.510200:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203932.510202:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203932.510203:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203932.510206:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203932.510211:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365376:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203932.510213:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.510215:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.510216:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543365248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.510220:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203932.510222:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203932.510223:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203932.510225:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203932.510226:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203932.510227:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203932.510228:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203932.510229:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:0.0:1788203932.510231:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203932.510232:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203932.510233:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:0.0:1788203932.510235:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203932.510236:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.510237:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203932.510239:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543365376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203957 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203932.510244:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203932.510245:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203932.510246:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203932.510247:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203932.510248:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203932.510248:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203932.510249:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203932.510250:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:0.0:1788203932.510251:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203932.510252:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203932.510253:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:0.0:1788203932.510254:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203932.510255:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.510256:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203932.510258:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203932.510259:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203932.510260:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203932.510261:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203933.340156:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203933.340164:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203933.340169:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203933.340175:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203933.340176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203933.340183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203933.340184:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203933.340190:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203933.340193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203933.340197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203933.340197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203933.340202:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203933.340204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203933.340206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203933.340209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203933.340216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203933.340217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203933.852186:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320292613, at 4320292672 00000100:00000010:2.0:1788203933.852206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fab40. 00000100:00000001:2.0:1788203934.364184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203934.364193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203934.364197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203934.364198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203934.364203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203934.364204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203934.364209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203934.364213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203934.364214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203934.364218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203934.364219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203934.364222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203934.364227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203934.364229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203934.364235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203934.364239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203934.364240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203935.388190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203935.388194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203935.388198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203935.388201:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788203935.388201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203935.388205:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788203935.388209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203935.388210:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203935.388216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203935.388217:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203935.388220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203935.388221:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203935.388222:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203935.388243:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203935.388248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203935.388248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203935.388250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203935.388253:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203935.388254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203935.388256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203935.388261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203935.388262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203935.388262:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203936.412192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203936.412196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203936.412201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203936.412203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203936.412205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203936.412226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203936.412228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203936.412231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203936.412234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203936.412240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203936.412240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203936.412245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203936.412248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203936.412252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203936.412254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203936.412255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203936.412261:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203937.436219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203937.436225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203937.436228:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203937.436231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203937.436233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203937.436240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203937.436240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203937.436245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203937.436247:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203937.436251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203937.436252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203937.436257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203937.436257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203937.436260:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203937.436263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203937.436264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203937.436268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.628431:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203937.628436:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203937.628440:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203937.628449:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.628462:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203937.628465:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.628470:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203937.628474:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203937.628478:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203937.628485:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543365504:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203937.628492:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203937.628497:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203937.628501:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.628505:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203937.628510:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203937.628516:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203937.628521:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203937.628524:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203937.628532:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203937.628542:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543365504, portal 25 00000100:00000001:0.0:1788203937.628550:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203937.628554:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:0.0:1788203937.628567:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.628584:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203937.628590:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203937.628596:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203937.628604:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543365504, offset 0 00000400:00000200:0.0:1788203937.628617:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203937.628626:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.628635:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203937.628642:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203937.628648:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203937.628653:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203937.628656:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203937.628663:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470204:470204:256:4294967295] 192.168.201.118@tcp LPNI seq info [470063:470063:8:4294967295] 00000400:00000200:0.0:1788203937.628677:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.628695:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203937.628707:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.628716:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203937.628730:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203937.628738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203937.628749:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203937.628754:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788203937.628767:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.628773:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203937.628785:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203937.628797:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203937.628807:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203937.628813:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203937.628821:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203937.628827:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.628840:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.628847:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.628856:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.628859:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203937.628866:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203937.628872:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203937.628878:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320297733 00000100:00000010:0.0:1788203937.628884:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa5a0. 00000100:00000001:0.0:1788203937.628892:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788203937.628900:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320297733 00000100:00000010:0.0:1788203937.628906:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa420. 00000100:00000001:0.0:1788203937.628910:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788203937.628916:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203937.628919:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.628923:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203937.628929:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203937/real 1788203937] req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203937.628936:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203937.628938:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203937.628942:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203937.628945:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203937.628947:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203937.628953:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203937.628958:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203937.628971:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.628982:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203937.628985:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.628988:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.628993:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203937.629000:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629008:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203937.629014:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629026:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203937.629031:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629035:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.629038:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629042:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203937.629045:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203937.629050:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203937.629058:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203937.629063:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203937.629068:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203937.629078:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543365504:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203937.629083:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629086:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.629119:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543365504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629138:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203937.629143:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203937.629148:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203937.629153:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203937.629156:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203937.629159:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788203937.629161:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203937.629165:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:0.0:1788203937.629170:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203937.629173:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203937.629176:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:0.0:1788203937.629182:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203937.629184:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629188:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629212:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203937.629215:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.629219:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203937.629222:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203937.629224:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203937.629231:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629245:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203937.629248:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.629252:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203937.629256:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203937.629260:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203937.629266:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365632:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203937.629273:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203937.629276:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203937.629279:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.629283:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203937.629288:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203937.629293:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203937.629298:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203937.629303:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203937.629309:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203937.629314:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543365632, portal 10 00000100:00000001:0.0:1788203937.629319:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203937.629322:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:0.0:1788203937.629332:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629347:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203937.629351:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203937.629356:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203937.629361:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543365632, offset 0 00000400:00000200:0.0:1788203937.629366:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203937.629372:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.629378:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203937.629383:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203937.629386:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203937.629388:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203937.629393:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203937.629399:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470205:470205:256:4294967295] 192.168.201.118@tcp LPNI seq info [470064:470064:8:4294967295] 00000400:00000200:0.0:1788203937.629407:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.629418:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203937.629424:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.629428:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203937.629435:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203937.629438:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203937.629447:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203937.629452:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788203937.629460:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.629462:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203937.629466:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203937.629471:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203937.629474:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203937.629477:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203937.629483:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203937.629490:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629505:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.629511:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629518:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.629520:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203937.629525:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203937.629530:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788203937.629534:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203937.629537:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320296467 00000100:00000001:0.0:1788203937.629540:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320297733 00000100:00000001:0.0:1788203937.629542:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320296467 00000100:00000001:0.0:1788203937.629545:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320297733 00000100:00000001:0.0:1788203937.629547:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203937.629549:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.629551:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203937.629556:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203937/real 1788203937] req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203937.629565:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203937.629566:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203937.629568:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203937.629571:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203937.629572:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203937.629575:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203937.629578:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203937.629584:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629588:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203937.629589:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629591:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.629592:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203937.629594:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629598:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203937.629601:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629605:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203937.629607:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629608:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.629608:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629609:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203937.629611:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203937.629613:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203937.629616:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203937.629617:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203937.629620:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203937.629625:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365632:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203937.629627:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203937.629629:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629633:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203937.629634:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.629636:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203937.629637:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203937.629639:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203937.629641:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365760:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203937.629643:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203937.629644:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203937.629645:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.629646:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203937.629649:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203937.629650:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203937.629653:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203937.629656:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203937.629659:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203937.629662:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543365760, portal 10 00000100:00000001:0.0:1788203937.629664:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203937.629666:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:0.0:1788203937.629672:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629685:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203937.629687:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203937.629689:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203937.629692:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543365760, offset 0 00000400:00000200:0.0:1788203937.629696:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203937.629701:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.629705:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203937.629709:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203937.629712:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203937.629715:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203937.629717:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203937.629720:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470206:470206:256:4294967295] 192.168.201.118@tcp LPNI seq info [470065:470065:8:4294967295] 00000400:00000200:0.0:1788203937.629724:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.629731:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203937.629736:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.629739:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203937.629743:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203937.629746:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203937.629750:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203937.629753:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788203937.629757:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.629759:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203937.629763:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203937.629767:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203937.629770:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203937.629772:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203937.629775:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203937.629779:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629788:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.629791:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629801:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.629803:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203937.629807:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203937.629810:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:0.0:1788203937.629813:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203937.629815:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320296468 00000100:00000001:0.0:1788203937.629818:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320297733 00000100:00000001:0.0:1788203937.629821:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320296468 00000100:00000001:0.0:1788203937.629823:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320297733 00000100:00000001:0.0:1788203937.629824:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203937.629826:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.629828:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203937.629832:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203937/real 1788203937] req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203937.629839:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203937.629841:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203937.629842:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203937.629845:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203937.629846:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203937.629850:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203937.629853:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203937.629862:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629868:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203937.629869:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629871:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.629872:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203937.629876:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629884:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203937.629888:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629894:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203937.629896:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629898:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.629899:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629901:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203937.629903:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203937.629906:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203937.629909:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203937.629912:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203937.629915:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203937.629923:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365760:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203937.629926:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629928:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.629931:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543365632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629937:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203937.629938:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203937.629940:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203937.629942:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203937.629944:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203937.629945:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203937.629947:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203937.629948:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:0.0:1788203937.629951:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203937.629952:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203937.629954:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:0.0:1788203937.629956:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203937.629957:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629959:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.629961:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543365760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.629967:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203937.629968:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203937.629970:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203937.629972:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203937.629973:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203937.629974:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203937.629976:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203937.629977:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:0.0:1788203937.629979:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203937.629980:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203937.629981:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:0.0:1788203937.629984:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203937.629985:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629986:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.629992:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203937.629993:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.629995:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203937.629996:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203937.629997:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203937.630000:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630006:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203937.630008:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.630010:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203937.630011:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203937.630013:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203937.630016:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365888:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203937.630021:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203937.630023:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203937.630025:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.630027:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203937.630030:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203937.630031:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203937.630036:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203937.630039:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203937.630042:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203937.630045:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543365888, portal 4 00000100:00000001:0.0:1788203937.630047:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203937.630048:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:0.0:1788203937.630054:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630060:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203937.630062:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203937.630065:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203937.630067:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543365888, offset 0 00000400:00000200:0.0:1788203937.630071:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203937.630075:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.630078:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203937.630081:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203937.630083:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203937.630086:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203937.630088:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203937.630109:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470207:470207:256:4294967295] 192.168.201.118@tcp LPNI seq info [470066:470066:8:4294967295] 00000400:00000200:0.0:1788203937.630112:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.630119:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203937.630124:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.630128:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203937.630132:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203937.630135:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203937.630140:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203937.630143:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788203937.630147:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.630149:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203937.630152:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203937.630156:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203937.630159:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203937.630161:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203937.630164:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203937.630169:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630178:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.630181:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630188:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.630203:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203937.630207:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203937.630210:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:0.0:1788203937.630214:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203937.630216:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320296478 00000100:00000001:0.0:1788203937.630219:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320297733 00000100:00000001:0.0:1788203937.630221:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320296478 00000100:00000001:0.0:1788203937.630224:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320297733 00000100:00000001:0.0:1788203937.630226:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203937.630229:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.630230:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203937.630235:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203937/real 1788203937] req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203937.630243:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203937.630245:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203937.630247:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203937.630250:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203937.630252:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203937.630255:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203937.630259:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203937.630268:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630275:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203937.630277:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630279:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.630280:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203937.630284:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630292:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203937.630296:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630304:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203937.630306:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630308:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.630309:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630311:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203937.630313:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203937.630316:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203937.630319:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203937.630322:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203937.630327:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203937.630335:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543365888:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203937.630340:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203937.630343:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630350:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203937.630351:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.630353:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203937.630355:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203937.630356:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203937.630358:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366016:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203937.630360:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203937.630362:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203937.630364:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203937.630366:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203937.630369:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203937.630372:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203937.630375:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203937.630378:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203937.630382:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203937.630385:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543366016, portal 4 00000100:00000001:0.0:1788203937.630387:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203937.630389:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:0.0:1788203937.630395:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630402:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203937.630404:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203937.630407:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203937.630409:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543366016, offset 0 00000400:00000200:0.0:1788203937.630414:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203937.630418:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.630422:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203937.630426:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203937.630429:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203937.630431:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203937.630434:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203937.630437:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470208:470208:256:4294967295] 192.168.201.118@tcp LPNI seq info [470067:470067:8:4294967295] 00000400:00000200:0.0:1788203937.630441:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203937.630449:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203937.630454:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.630458:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203937.630462:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203937.630465:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:0.1:1788203937.630469:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203937.630472:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:0.0:1788203937.630476:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203937.630477:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203937.630481:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203937.630483:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203937.630486:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203937.630487:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203937.630489:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203937.630492:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630498:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.630500:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203962 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630504:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.630505:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203937.630507:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203937.630509:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:0.0:1788203937.630511:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203937.630513:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320296501 00000100:00000001:0.0:1788203937.630515:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320297733 00000100:00000001:0.0:1788203937.630516:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320296501 00000100:00000001:0.0:1788203937.630518:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320297733 00000100:00000001:0.0:1788203937.630519:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203937.630521:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.630522:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203937.630525:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203937/real 1788203937] req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203937.630530:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203937.630531:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203937.630533:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203937.630536:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203937.630538:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203937.630540:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203937.630543:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203937.630552:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630559:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203937.630561:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630563:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.630564:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203937.630568:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630575:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203937.630579:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630586:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203937.630588:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630589:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203937.630590:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630592:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203937.630593:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203937.630595:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203937.630597:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203937.630599:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203937.630602:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203937.630607:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366016:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203937.630609:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630610:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.630612:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543365888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630616:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203937.630617:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203937.630618:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203937.630620:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203937.630622:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203937.630623:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203937.630624:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203937.630625:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:0.0:1788203937.630627:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203937.630628:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203937.630629:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:0.0:1788203937.630631:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203937.630632:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630633:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203937.630634:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543366016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203962 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203937.630638:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203937.630639:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203937.630640:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203937.630641:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203937.630643:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203937.630643:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203937.630645:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203937.630646:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:0.0:1788203937.630647:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203937.630648:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203937.630649:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:0.0:1788203937.630650:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203937.630651:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630652:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203937.630655:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203937.630656:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203937.630657:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203937.630658:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203938.460218:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203938.460219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203938.460226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203938.460227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203938.460231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203938.460236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203938.460239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203938.460240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203938.460247:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203938.460249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203938.460253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203938.460255:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203938.460258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203938.460262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203938.460263:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203938.460269:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203938.460277:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203938.972213:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320297733, at 4320297792 00000100:00000010:2.0:1788203938.972229:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa5a0. 00010000:00000001:0.0:1788203939.484204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203939.484207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203939.484220:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203939.484223:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203939.484226:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484230:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203939.484233:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484241:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203939.484251:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203939.484252:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788203939.484257:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203939.484263:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203939.484264:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203939.484268:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484270:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:2.0:1788203939.484271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203939.484273:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203939.484275:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203939.484276:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484279:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203939.484279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203939.484286:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203939.484287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203939.484291:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788203939.484292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203939.484293:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203939.484295:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484298:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788203939.484299:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484300:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484305:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203939.484308:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203940.508202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788203940.508211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203940.508214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203940.508214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203940.508219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203940.508220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788203940.508225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203940.508226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203940.508228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203940.508229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203940.508233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203940.508237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203940.508237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203940.508241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203940.508244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203940.508246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203940.508251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203941.532220:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203941.532222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203941.532233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203941.532234:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203941.532239:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203941.532247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203941.532252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203941.532253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203941.532262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203941.532264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203941.532271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203941.532274:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203941.532283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203941.532283:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203941.532289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203941.532289:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203941.532297:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203942.556199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203942.556203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203942.556207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203942.556208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203942.556211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203942.556216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203942.556218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203942.556220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203942.556221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203942.556226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203942.556230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203942.556230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203942.556235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203942.556235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203942.556244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203942.556246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203942.556250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.748552:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203942.748559:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203942.748564:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203942.748574:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.748589:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203942.748594:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.748599:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203942.748602:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203942.748608:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203942.748613:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543366144:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203942.748620:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203942.748625:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203942.748629:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.748633:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203942.748640:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203942.748645:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203942.748657:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203942.748662:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203942.748675:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203942.748686:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543366144, portal 25 00000100:00000001:0.0:1788203942.748691:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203942.748693:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:0.0:1788203942.748703:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.748739:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203942.748747:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203942.748754:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203942.748763:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543366144, offset 0 00000400:00000200:0.0:1788203942.748779:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203942.748799:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.748810:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203942.748821:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203942.748830:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203942.748837:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203942.748845:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203942.748857:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470209:470209:256:4294967295] 192.168.201.118@tcp LPNI seq info [470068:470068:8:4294967295] 00000400:00000200:0.0:1788203942.748869:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.748887:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203942.748898:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.748906:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203942.748921:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203942.748926:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203942.748941:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203942.748947:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788203942.748956:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.748960:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203942.748968:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203942.748977:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203942.748985:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203942.748990:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203942.748996:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203942.749003:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749023:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.749028:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749042:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.749046:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203942.749054:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203942.749064:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203942.749069:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320302853 00000100:00000010:0.0:1788203942.749078:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fae40. 00000100:00000001:0.0:1788203942.749088:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788203942.749143:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320302854 00000100:00000010:0.0:1788203942.749149:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa6c0. 00000100:00000001:0.0:1788203942.749155:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788203942.749160:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203942.749166:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.749172:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203942.749179:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203942/real 1788203942] req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203942.749197:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203942.749200:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203942.749204:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203942.749212:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203942.749215:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203942.749219:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203942.749226:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203942.749242:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749259:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203942.749262:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749267:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.749270:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203942.749277:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749292:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203942.749300:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749312:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203942.749317:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749322:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.749326:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749331:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203942.749334:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203942.749338:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203942.749344:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203942.749347:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203942.749355:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203942.749385:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543366144:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203942.749393:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749397:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.749405:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543366144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749421:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203942.749427:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203942.749431:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203942.749435:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203942.749437:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203942.749439:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788203942.749442:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203942.749445:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:0.0:1788203942.749450:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203942.749453:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203942.749455:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:0.0:1788203942.749461:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203942.749463:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749465:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749475:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203942.749477:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.749479:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203942.749481:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203942.749482:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203942.749485:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749491:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203942.749492:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.749494:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203942.749495:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203942.749497:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203942.749500:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366272:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203942.749502:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203942.749504:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203942.749505:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.749507:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203942.749509:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203942.749510:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203942.749513:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203942.749515:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203942.749518:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203942.749521:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543366272, portal 10 00000100:00000001:0.0:1788203942.749522:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203942.749523:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000040:0.0:1788203942.749529:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749534:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203942.749535:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203942.749537:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203942.749538:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543366272, offset 0 00000400:00000200:0.0:1788203942.749541:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203942.749544:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.749547:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203942.749549:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203942.749551:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203942.749553:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203942.749555:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203942.749557:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470210:470210:256:4294967295] 192.168.201.118@tcp LPNI seq info [470069:470069:8:4294967295] 00000400:00000200:0.0:1788203942.749561:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.749565:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203942.749569:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.749571:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203942.749574:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203942.749577:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203942.749580:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203942.749581:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788203942.749584:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.749586:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203942.749588:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203942.749590:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203942.749592:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203942.749594:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203942.749596:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203942.749598:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749604:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.749606:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749611:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.749612:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203942.749614:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203942.749618:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788203942.749620:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203942.749621:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320301587 00000100:00000001:0.0:1788203942.749623:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320302853 00000100:00000001:0.0:1788203942.749625:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320301587 00000100:00000001:0.0:1788203942.749626:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320302854 00000100:00000001:0.0:1788203942.749629:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203942.749631:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.749633:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203942.749637:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203942/real 1788203942] req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203942.749653:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203942.749654:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203942.749656:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203942.749659:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203942.749661:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203942.749663:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203942.749667:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203942.749676:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749683:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203942.749685:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749687:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.749688:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203942.749692:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749700:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203942.749705:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749713:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203942.749716:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749718:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.749719:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749722:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203942.749724:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203942.749727:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203942.749731:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203942.749734:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203942.749739:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203942.749748:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366272:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203942.749751:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203942.749753:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749758:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203942.749759:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.749761:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203942.749762:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203942.749764:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203942.749766:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366400:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203942.749768:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203942.749770:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203942.749771:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.749772:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203942.749775:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203942.749777:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203942.749779:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203942.749781:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203942.749783:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203942.749786:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543366400, portal 10 00000100:00000001:0.0:1788203942.749787:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203942.749788:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:0.0:1788203942.749793:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749798:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203942.749799:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203942.749801:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203942.749802:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543366400, offset 0 00000400:00000200:0.0:1788203942.749805:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203942.749808:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.749811:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203942.749813:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203942.749816:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203942.749819:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203942.749821:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203942.749825:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470211:470211:256:4294967295] 192.168.201.118@tcp LPNI seq info [470070:470070:8:4294967295] 00000400:00000200:0.0:1788203942.749830:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.749837:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203942.749842:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.749846:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203942.749851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203942.749854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203942.749859:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203942.749861:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788203942.749865:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.749867:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203942.749871:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203942.749873:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203942.749876:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203942.749878:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203942.749880:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203942.749883:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749889:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.749891:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749895:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.749897:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203942.749899:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203942.749901:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:0.0:1788203942.749904:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203942.749906:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320301588 00000100:00000001:0.0:1788203942.749908:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320302853 00000100:00000001:0.0:1788203942.749910:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320301588 00000100:00000001:0.0:1788203942.749911:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320302854 00000100:00000001:0.0:1788203942.749912:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203942.749914:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.749915:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203942.749917:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203942/real 1788203942] req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203942.749923:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203942.749924:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203942.749925:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203942.749926:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203942.749928:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203942.749929:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203942.749931:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203942.749937:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749941:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203942.749941:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749942:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.749943:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203942.749945:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749949:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203942.749952:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749956:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203942.749958:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749959:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.749960:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.749961:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203942.749963:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203942.749965:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203942.749967:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203942.749968:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203942.749971:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203942.749976:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366400:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203942.749979:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203942.749981:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.749985:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203942.749986:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.749987:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203942.749988:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203942.749990:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203942.749991:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366528:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203942.749993:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203942.749994:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203942.749995:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.749997:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203942.749999:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28800. 02000000:00000001:0.0:1788203942.750000:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203942.750003:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203942.750005:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203942.750007:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203942.750009:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543366528, portal 4 00000100:00000001:0.0:1788203942.750011:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203942.750012:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:0.0:1788203942.750016:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750021:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203942.750022:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203942.750024:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203942.750025:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543366528, offset 0 00000400:00000200:0.0:1788203942.750028:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203942.750030:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.750032:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203942.750035:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203942.750036:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203942.750038:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203942.750040:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203942.750042:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470212:470212:256:4294967295] 192.168.201.118@tcp LPNI seq info [470071:470071:8:4294967295] 00000400:00000200:0.0:1788203942.750045:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.750050:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203942.750052:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.750054:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203942.750057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203942.750059:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203942.750062:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203942.750064:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788203942.750066:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.750068:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203942.750070:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203942.750072:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203942.750074:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203942.750075:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203942.750077:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203942.750079:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750121:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.750125:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750130:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.750131:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203942.750134:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203942.750136:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:0.0:1788203942.750139:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203942.750140:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320301598 00000100:00000001:0.0:1788203942.750142:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320302853 00000100:00000001:0.0:1788203942.750144:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320301598 00000100:00000001:0.0:1788203942.750145:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320302854 00000100:00000001:0.0:1788203942.750147:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203942.750148:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.750149:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203942.750151:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203942/real 1788203942] req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203942.750156:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203942.750157:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203942.750159:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203942.750160:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203942.750162:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203942.750163:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203942.750166:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203942.750171:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750176:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203942.750177:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750178:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.750180:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203942.750182:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750186:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203942.750189:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750193:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203942.750194:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750195:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.750196:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750197:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203942.750198:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203942.750200:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203942.750201:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203942.750203:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203942.750205:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203942.750210:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366528:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203942.750212:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203942.750214:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750219:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203942.750220:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.750221:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203942.750223:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203942.750224:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203942.750226:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366656:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203942.750228:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203942.750230:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203942.750231:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203942.750232:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203942.750235:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c29400. 02000000:00000001:0.0:1788203942.750236:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203942.750238:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203942.750240:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203942.750241:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203942.750243:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543366656, portal 4 00000100:00000001:0.0:1788203942.750245:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203942.750246:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:0.0:1788203942.750250:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750255:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203942.750256:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203942.750258:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203942.750259:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543366656, offset 0 00000400:00000200:0.0:1788203942.750261:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203942.750264:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.750266:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203942.750268:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203942.750270:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203942.750272:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203942.750274:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203942.750276:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470213:470213:256:4294967295] 192.168.201.118@tcp LPNI seq info [470072:470072:8:4294967295] 00000400:00000200:0.0:1788203942.750279:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203942.750283:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203942.750286:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.750288:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203942.750291:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203942.750294:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:0.1:1788203942.750296:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203942.750298:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:0.0:1788203942.750301:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203942.750302:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203942.750304:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203942.750306:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203942.750308:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203942.750310:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203942.750311:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203942.750313:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750318:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.750320:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203967 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750325:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.750326:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203942.750328:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203942.750329:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:0.0:1788203942.750331:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203942.750332:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320301621 00000100:00000001:0.0:1788203942.750334:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320302853 00000100:00000001:0.0:1788203942.750335:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320301621 00000100:00000001:0.0:1788203942.750337:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320302854 00000100:00000001:0.0:1788203942.750339:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203942.750340:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.750341:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203942.750343:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203942/real 1788203942] req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203942.750347:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203942.750349:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203942.750350:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203942.750352:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203942.750353:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203942.750354:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203942.750356:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203942.750361:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750365:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203942.750366:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750367:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.750367:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203942.750370:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750373:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203942.750376:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750380:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203942.750381:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750382:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203942.750383:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750384:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203942.750385:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203942.750387:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203942.750390:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203942.750391:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203942.750394:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203942.750403:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543366656:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203942.750405:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750407:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.750409:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543366272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750413:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203942.750414:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203942.750415:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:0.0:1788203942.750417:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203942.750418:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203942.750418:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203942.750421:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203942.750423:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:0.0:1788203942.750425:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203942.750427:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203942.750428:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:0.0:1788203942.750435:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203942.750437:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750438:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.750442:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543366400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750449:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203942.750450:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203942.750452:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c29c00. 02000000:00000001:0.0:1788203942.750455:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203942.750456:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203942.750458:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203942.750460:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203942.750462:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:0.0:1788203942.750464:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203942.750466:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203942.750467:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:0.0:1788203942.750475:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203942.750476:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750479:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.750481:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543366528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750486:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203942.750487:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203942.750488:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28800. 02000000:00000001:0.0:1788203942.750490:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203942.750491:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203942.750492:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203942.750493:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203942.750494:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:0.0:1788203942.750495:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203942.750496:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203942.750497:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:0.0:1788203942.750498:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203942.750499:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750500:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203942.750502:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543366656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203967 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203942.750506:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203942.750507:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203942.750508:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c29400. 02000000:00000001:0.0:1788203942.750510:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203942.750510:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203942.750511:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203942.750512:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203942.750513:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:0.0:1788203942.750514:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203942.750515:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203942.750515:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:0.0:1788203942.750517:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203942.750518:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750519:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203942.750522:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203942.750523:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203942.750525:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203942.750526:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203943.580231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203943.580234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203943.580240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203943.580240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203943.580247:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203943.580253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203943.580254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203943.580257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203943.580263:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203943.580264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203943.580269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203943.580269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203943.580278:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203943.580279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203943.580282:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203943.580286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203943.580293:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203944.092194:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320302853, at 4320302912 00000100:00000010:2.0:1788203944.092209:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fae40. 00000100:00000001:2.0:1788203944.604239:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203944.604240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203944.604248:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203944.604253:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788203944.604253:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203944.604263:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203944.604264:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203944.604271:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203944.604276:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203944.604278:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203944.604280:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203944.604292:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203944.604294:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203944.604298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203944.604301:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203944.604305:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203944.604307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203944.604312:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203944.604317:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203944.604319:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203944.604324:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203944.604326:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203944.604334:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203945.628231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203945.628232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203945.628242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203945.628243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203945.628248:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203945.628256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203945.628257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203945.628262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203945.628269:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203945.628272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203945.628276:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203945.628277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203945.628280:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203945.628285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203945.628286:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203945.628294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203945.628299:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203946.652211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203946.652218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203946.652230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203946.652235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203946.652243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203946.652248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203946.652255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203946.652259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203947.676241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203947.676247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203947.676261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203947.676267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203947.676276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203947.676285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203947.676293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203947.676299:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203947.676404:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203947.676408:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203947.676411:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203947.868237:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203947.868253:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203947.868257:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203947.868261:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203947.868267:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203947.868276:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25634 00000400:00000200:0.0:1788203947.868290:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203947.868300:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203947.868303:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203947.868306:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868310:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.868313:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868315:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203947.868320:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203947.868323:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203947.868327:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203947.868332:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203947.868334:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203947.868336:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203947.868340:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868342:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.868345:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868348:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203947.868351:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203947.868355:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868359:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203947.868363:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203947.868367:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203947.868371:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868374:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203947.868377:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203947.868390:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788203947.868395:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203947.868401:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203947.868403:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868407:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203947.868411:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203947.868417:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203947.868429:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788203947.868437:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203947.868449:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543366784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203947.868461:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203947.868465:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203947.868470:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543366784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203947.868498:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203947.868500:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203947.868504:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203947.868505:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203947.868506:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203947.868509:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203947.868511:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25634 00000400:00000200:0.0:1788203947.868515:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203947.868518:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203947.868520:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203947.868522:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868524:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.868526:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868528:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203947.868530:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203947.868531:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203947.868532:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203947.868534:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203947.868536:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203947.868537:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203947.868540:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868541:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.868543:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868544:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203947.868546:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203947.868549:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868553:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203947.868555:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203947.868561:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203947.868564:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868565:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203947.868570:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203947.868576:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868580:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203947.868588:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788203947.868593:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203947.868598:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203947.868613:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868617:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203947.868622:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203947.868626:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203947.868634:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788203947.868642:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203947.868657:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543366912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203947.868665:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203947.868667:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203947.868674:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543366912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203947.868682:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203947.868688:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203947.868693:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203947.868696:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203947.868697:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203947.868701:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203947.868703:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25634 00000400:00000200:0.0:1788203947.868708:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203947.868712:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203947.868715:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203947.868720:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868725:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.868729:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868733:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203947.868741:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203947.868744:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203947.868747:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203947.868749:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203947.868752:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203947.868755:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203947.868759:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868761:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.868762:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868763:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203947.868765:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203947.868768:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868771:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203947.868772:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203947.868774:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203947.868776:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868777:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203947.868779:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203947.868783:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868786:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203947.868790:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788203947.868793:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203947.868798:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203947.868800:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868802:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203947.868806:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203947.868807:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203947.868810:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788203947.868815:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203947.868821:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543367040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203947.868837:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203947.868842:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203947.868849:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543367040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203947.868860:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203947.868865:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203947.868870:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788203947.868873:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203947.868875:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203947.868877:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203947.868882:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25634 00000400:00000200:0.0:1788203947.868888:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203947.868893:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203947.868896:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203947.868902:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868904:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.868911:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868917:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203947.868923:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203947.868927:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000800:00000200:3.1:1788203947.868928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:0.0:1788203947.868930:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:3.1:1788203947.868933:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000020:00000040:0.0:1788203947.868935:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203947.868941:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203947.868947:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203947.868954:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868959:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.868964:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868968:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203947.868972:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203947.868978:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203947.868982:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203947.868985:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203947.868988:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203947.868992:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.868996:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203947.869000:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203947.869006:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.869008:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203947.869028:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788203947.869032:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203947.869035:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203947.869038:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.869042:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203947.869045:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203947.869047:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203947.869053:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788203947.869059:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203947.869065:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543367168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203947.869080:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203947.869085:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203947.869148:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543367168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203947.869178:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203947.869184:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203947.869193:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203947.869199:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203947.869203:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203947.869210:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203947.869217:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25634 00000400:00000200:0.0:1788203947.869229:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203947.869239:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203947.869245:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203947.869255:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.869263:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.869270:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.869279:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203947.869286:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203947.869291:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203947.869296:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203947.869301:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203947.869309:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203947.869316:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203947.869325:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.869329:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203947.869335:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203947.869341:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203947.869349:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203947.869356:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203947.869361:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203947.869368:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203947.869374:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203947.869379:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.869384:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203947.869388:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203947.869396:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.869401:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203947.869409:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788203947.869416:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203947.869422:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203947.869425:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203947.869427:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203947.869429:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203947.869431:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203947.869438:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788203947.869445:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203947.869455:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543367296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203947.869493:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203947.869499:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203947.869507:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543367296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203947.869523:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203947.869529:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25644) 00000800:00000200:3.1:1788203947.869629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203947.869632:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203947.869843:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203947.869845:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203947.870108:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203947.870110:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788203947.870426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203947.870428:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788203948.700242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203948.700250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203948.700262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203948.700266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203948.700271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203948.700276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203948.700286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203948.700290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203948.700334:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203948.700341:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203948.700345:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203949.212226:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320307973, at 4320308032 00000100:00000010:2.0:1788203949.212248:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcd20. 00010000:00000001:0.0:1788203949.724179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203949.724184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203949.724197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203949.724202:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203949.724209:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724213:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203949.724216:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724221:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724226:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203949.724236:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203949.724237:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203949.724238:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724240:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203949.724241:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724243:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724245:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203949.724249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203949.724250:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203949.724251:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724252:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203949.724253:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203949.724255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203950.748224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203950.748230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203950.748239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203950.748243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203950.748249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203950.748253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203950.748259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203950.748263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203950.748336:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203950.748340:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203950.748343:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203951.772197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203951.772203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203951.772213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203951.772217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203951.772226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203951.772230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203951.772235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203951.772238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203951.772344:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203951.772347:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203951.772350:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203952.796212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203952.796219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203952.796230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203952.796235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203952.796244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203952.796249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203952.796254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203952.796258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203952.796305:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203952.796310:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203952.796313:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203952.988214:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.988223:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203952.988225:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203952.988228:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203952.988231:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203952.988237:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25639 00000400:00000200:2.0:1788203952.988248:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203952.988255:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203952.988259:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203952.988263:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988266:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.988269:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988272:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203952.988276:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203952.988278:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203952.988281:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203952.988285:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203952.988288:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203952.988291:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203952.988294:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988297:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.988298:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988300:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203952.988302:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203952.988304:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988306:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203952.988308:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203952.988311:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203952.988313:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.988315:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203952.988317:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203952.988325:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203952.988328:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203952.988331:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203952.988332:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.988335:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203952.988337:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203952.988341:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203952.988347:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788203952.988352:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203952.988359:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543367424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203952.988368:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203952.988373:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203952.988378:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543367424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203952.988402:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203952.988405:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.988409:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203952.988411:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203952.988412:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203952.988416:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203952.988420:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25639 00000400:00000200:2.0:1788203952.988425:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203952.988429:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203952.988431:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203952.988434:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988438:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.988440:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988444:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203952.988447:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203952.988448:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203952.988451:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203952.988453:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203952.988457:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203952.988462:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203952.988468:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988471:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.988475:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988478:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203952.988483:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203952.988491:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988497:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203952.988503:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203952.988511:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203952.988516:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.988524:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203952.988531:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203952.988538:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.988541:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203952.988550:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788203952.988557:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203952.988565:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203952.988569:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.988575:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203952.988578:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203952.988581:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203952.988588:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788203952.988594:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203952.988602:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543367552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203952.988615:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203952.988621:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203952.988630:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543367552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203952.988653:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203952.988660:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.988669:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203952.988674:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203952.988679:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203952.988684:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203952.988691:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25639 00000400:00000200:2.0:1788203952.988701:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.1:1788203952.988707:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203952.988711:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000400:00000200:2.0:1788203952.988712:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203952.988717:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203952.988725:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988728:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.988731:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988733:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203952.988738:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203952.988741:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203952.988744:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203952.988747:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203952.988749:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203952.988752:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203952.988758:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988761:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.988768:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988774:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203952.988781:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203952.988790:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203952.988797:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203952.988801:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203952.988808:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203952.988812:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.988815:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203952.988819:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203952.988825:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.988831:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203952.988844:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788203952.988850:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203952.988855:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203952.988858:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.988864:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203952.988866:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203952.988870:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203952.988877:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788203952.988885:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203952.988895:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543367680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203952.988941:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203952.988944:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203952.988951:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543367680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203952.988971:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203952.988977:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.988987:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203952.988991:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203952.988995:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203952.989002:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203952.989010:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25639 00000400:00000200:2.0:1788203952.989020:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203952.989031:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203952.989038:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203952.989046:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989051:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.989058:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989062:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203952.989066:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203952.989069:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203952.989075:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203952.989081:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203952.989087:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203952.989124:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203952.989134:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989141:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.989147:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989152:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203952.989157:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203952.989166:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989174:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203952.989179:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203952.989186:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203952.989193:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.989200:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203952.989206:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203952.989212:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.989218:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203952.989226:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788203952.989232:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203952.989237:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203952.989242:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.989249:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203952.989254:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203952.989257:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203952.989263:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788203952.989270:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203952.989282:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543367808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203952.989300:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203952.989306:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203952.989316:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543367808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203952.989336:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203952.989342:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203952.989353:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203952.989358:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203952.989363:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203952.989371:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203952.989378:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25639 00000400:00000200:2.0:1788203952.989409:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203952.989419:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203952.989425:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1788203952.989426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203952.989431:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000040:2.0:1788203952.989434:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989440:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.989448:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989454:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203952.989458:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203952.989463:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203952.989468:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203952.989472:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203952.989477:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203952.989484:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203952.989492:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989498:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203952.989504:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989508:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203952.989513:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203952.989521:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203952.989528:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203952.989535:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203952.989541:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203952.989549:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.989555:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203952.989559:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203952.989566:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.989571:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203952.989579:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788203952.989586:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203952.989592:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203952.989596:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203952.989600:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203952.989604:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203952.989609:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203952.989617:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788203952.989625:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203952.989637:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543367936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203952.989659:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203952.989665:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203952.989676:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543367936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203952.989709:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203952.989719:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25649) 00000800:00000200:3.1:1788203952.989808:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203952.989811:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203952.990082:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203952.990084:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788203952.990293:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203952.990297:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788203953.820300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203953.820304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203953.820309:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203953.820311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203953.820314:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203953.820316:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203953.820319:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203953.820320:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203953.820376:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203953.820380:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203953.820383:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203954.332192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320313093, at 4320313152 00000100:00000010:2.0:1788203954.332210:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf60. 00010000:00000001:0.0:1788203954.844199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203954.844205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203954.844209:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203954.844213:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203954.844220:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203954.844224:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203954.844228:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203954.844235:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203954.844240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203954.844243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203954.844249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203954.844252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203954.844257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203954.844259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203954.844309:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203954.844315:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203954.844319:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203955.868212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203955.868219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203955.868226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203955.868235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203955.868242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203955.868246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203955.868252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203955.868257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203955.868309:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203955.868312:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203955.868316:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203956.892200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203956.892207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203956.892216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203956.892221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203956.892232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203956.892236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203956.892244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203956.892249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203956.892313:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203956.892316:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203956.892318:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203957.916185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203957.916192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203957.916203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203957.916209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203957.916214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203957.916218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203957.916225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203957.916229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203957.916283:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203957.916287:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203957.916289:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788203958.108152:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203958.108165:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788203958.108168:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203958.108171:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203958.108176:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203958.108184:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25644 00000400:00000200:2.0:1788203958.108194:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203958.108202:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203958.108206:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203958.108210:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108212:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.108214:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108216:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203958.108219:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203958.108220:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788203958.108222:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203958.108225:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788203958.108227:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788203958.108229:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788203958.108231:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108232:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.108233:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108235:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203958.108241:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788203958.108245:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108247:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203958.108249:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203958.108253:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203958.108255:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108256:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788203958.108259:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203958.108266:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788203958.108269:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788203958.108272:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203958.108274:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108277:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203958.108280:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203958.108285:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203958.108292:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788203958.108300:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203958.108308:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543368064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203958.108320:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203958.108326:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203958.108330:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543368064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203958.108350:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203958.108353:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203958.108356:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788203958.108357:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203958.108358:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203958.108360:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203958.108362:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25644 00000400:00000200:2.0:1788203958.108365:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203958.108368:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203958.108370:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203958.108372:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108373:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.108375:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108376:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203958.108378:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203958.108379:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788203958.108380:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203958.108382:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788203958.108384:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788203958.108385:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788203958.108388:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108389:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.108390:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108392:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203958.108394:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788203958.108396:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108398:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203958.108399:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203958.108404:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203958.108408:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108413:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203958.108416:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203958.108419:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108421:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203958.108426:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788203958.108431:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203958.108437:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203958.108440:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108445:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203958.108449:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203958.108454:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203958.108462:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788203958.108471:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203958.108483:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543368192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203958.108496:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203958.108502:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203958.108510:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543368192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203958.108528:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203958.108551:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203958.108562:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788203958.108567:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203958.108569:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203958.108574:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203958.108581:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25644 00000400:00000200:2.0:1788203958.108592:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203958.108598:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203958.108605:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203958.108614:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108619:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.108624:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108630:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203958.108635:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203958.108637:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788203958.108639:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203958.108644:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788203958.108649:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788203958.108651:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788203958.108659:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108665:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.108671:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108677:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203958.108681:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788203958.108686:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108692:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203958.108696:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203958.108702:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203958.108708:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108714:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203958.108718:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788203958.108723:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108729:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203958.108736:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788203958.108741:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788203958.108748:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203958.108752:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108756:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203958.108759:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203958.108762:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203958.108769:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788203958.108776:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203958.108781:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00100000:2.0:1788203958.108785:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543368320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788203958.108788:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788203958.108797:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203958.108802:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203958.108809:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543368320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203958.108818:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203958.108825:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203958.108829:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788203958.108832:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203958.108849:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203958.108852:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203958.108857:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25644 00000400:00000200:2.0:1788203958.108865:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203958.108872:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203958.108877:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203958.108886:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108891:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.108897:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108902:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203958.108907:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203958.108910:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788203958.108914:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203958.108918:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788203958.108922:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788203958.108926:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788203958.108935:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108938:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.108940:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108941:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203958.108944:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788203958.108949:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203958.108952:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203958.108954:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203958.108959:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203958.108962:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108966:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203958.108968:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788203958.108970:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108973:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203958.108978:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788203958.108981:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203958.108987:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203958.108988:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.108990:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203958.108991:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203958.108995:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203958.109000:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788203958.109008:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203958.109017:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543368448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203958.109029:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203958.109033:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203958.109040:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543368448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203958.109054:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788203958.109059:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788203958.109065:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788203958.109067:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788203958.109069:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788203958.109074:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788203958.109079:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25644 00000400:00000200:2.0:1788203958.109086:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203958.109119:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788203958.109123:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788203958.109127:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.109132:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.109134:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.109140:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788203958.109145:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788203958.109147:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788203958.109150:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788203958.109153:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788203958.109157:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788203958.109163:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788203958.109172:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.109177:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788203958.109184:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788203958.109190:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788203958.109195:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788203958.109202:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788203958.109207:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788203958.109213:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788203958.109217:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788203958.109221:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.109225:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788203958.109227:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788203958.109230:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.109232:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788203958.109237:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788203958.109241:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788203958.109245:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788203958.109248:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203958.109252:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788203958.109254:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788203958.109257:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788203958.109263:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788203958.109269:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203958.109281:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543368576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788203958.109299:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788203958.109303:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203958.109310:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543368576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203958.109331:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203958.109338:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25654) 00000800:00000200:3.1:1788203958.109512:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203958.109515:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203958.109731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203958.109733:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203958.110037:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203958.110039:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788203958.110248:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203958.110250:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788203958.940223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203958.940234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203958.940244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203958.940249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203958.940256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203958.940259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203958.940267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203958.940269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203958.940324:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203958.940328:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203958.940330:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203959.452176:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320318213, at 4320318272 00000100:00000010:2.0:1788203959.452192:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc300. 00010000:00000001:0.0:1788203959.964195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203959.964202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203959.964215:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203959.964218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203959.964222:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964226:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203959.964230:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203959.964246:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203959.964248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203959.964250:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964255:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203959.964257:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964262:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964265:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203959.964272:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203959.964273:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203959.964275:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964276:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203959.964277:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964278:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203959.964280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203959.964333:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203959.964337:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203959.964339:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203960.988169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203960.988175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203960.988183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203960.988187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203960.988194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203960.988199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203960.988206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203960.988209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203960.988256:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203960.988259:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203960.988262:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203962.012196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203962.012203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203962.012213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203962.012217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203962.012225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203962.012230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203962.012239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203962.012243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203962.012307:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203962.012310:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203962.012312:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203963.036214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203963.036222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203963.036232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203963.036236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203963.036241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203963.036245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203963.036252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203963.036256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203963.036378:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203963.036383:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203963.036386:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203963.228218:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203963.228230:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203963.228233:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203963.228237:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203963.228242:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203963.228248:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25649 00000400:00000200:0.0:1788203963.228259:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203963.228265:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203963.228269:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203963.228272:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228275:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.228278:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228280:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203963.228285:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203963.228287:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203963.228291:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203963.228295:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203963.228297:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203963.228300:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203963.228303:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228305:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.228307:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228308:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203963.228310:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203963.228313:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228314:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203963.228316:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203963.228319:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203963.228321:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.228322:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203963.228325:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203963.228333:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788203963.228335:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203963.228338:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203963.228340:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.228342:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203963.228345:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203963.228348:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203963.228355:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788203963.228360:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203963.228374:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543368704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203963.228383:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203963.228387:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203963.228390:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543368704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203963.228410:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203963.228412:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203963.228415:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203963.228416:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203963.228418:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203963.228420:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203963.228422:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25649 00000400:00000200:0.0:1788203963.228426:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203963.228428:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203963.228430:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203963.228432:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228433:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.228435:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228437:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203963.228439:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203963.228440:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203963.228441:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203963.228443:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203963.228444:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203963.228446:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203963.228448:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228450:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.228452:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228453:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203963.228455:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203963.228456:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228458:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203963.228460:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203963.228465:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203963.228469:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.228471:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203963.228475:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203963.228480:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.228483:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203963.228487:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788203963.228491:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203963.228496:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203963.228499:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.228502:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203963.228506:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203963.228509:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203963.228513:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788203963.228517:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203963.228527:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543368832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203963.228536:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203963.228540:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203963.228546:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543368832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203963.228556:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203963.228559:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203963.228561:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203963.228562:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203963.228563:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203963.228565:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203963.228567:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25649 00000400:00000200:0.0:1788203963.228569:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203963.228570:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203963.228573:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203963.228576:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228577:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.228581:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228586:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203963.228588:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203963.228592:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203963.228594:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203963.228595:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203963.228596:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203963.228598:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203963.228603:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228605:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.228607:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228609:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203963.228613:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203963.228619:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228624:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203963.228626:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203963.228630:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203963.228636:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.228639:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203963.228642:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203963.228646:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.228649:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203963.228656:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788203963.228659:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203963.228662:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203963.228664:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.228669:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203963.228670:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203963.228672:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203963.228676:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788203963.228680:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203963.228686:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543368960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203963.228707:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203963.228712:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203963.228720:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543368960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203963.228734:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203963.228738:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203963.228744:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:3.1:1788203963.228787:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203963.228790:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:0.0:1788203963.228901:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203963.228904:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203963.228910:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203963.228917:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25649 00000400:00000200:0.0:1788203963.228927:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203963.228932:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203963.228936:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203963.228942:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228947:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.228953:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.228958:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203963.228962:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203963.228965:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788203963.228968:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203963.228972:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203963.228979:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203963.228985:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203963.228995:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.229002:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.229010:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.229017:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203963.229025:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203963.229034:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203963.229042:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203963.229048:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203963.229055:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203963.229069:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.229073:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203963.229078:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203963.229083:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.229086:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203963.229135:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788203963.229141:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203963.229147:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203963.229151:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.229159:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203963.229162:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203963.229165:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203963.229171:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788203963.229179:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203963.229191:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543369088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203963.229212:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203963.229219:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203963.229230:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543369088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203963.229254:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203963.229259:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203963.229268:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203963.229273:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203963.229278:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203963.229286:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203963.229295:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25649 00000400:00000200:0.0:1788203963.229306:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203963.229316:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203963.229323:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203963.229333:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.229337:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.229344:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.229349:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203963.229356:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203963.229360:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203963.229365:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203963.229367:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203963.229372:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203963.229377:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203963.229385:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.229391:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203963.229397:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203963.229402:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203963.229408:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203963.229416:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203963.229422:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203963.229423:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:0.0:1788203963.229427:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.1:1788203963.229430:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 02000000:00000001:0.0:1788203963.229433:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203963.229438:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.229442:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203963.229447:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203963.229451:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.229454:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203963.229460:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788203963.229467:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203963.229473:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203963.229478:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203963.229483:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203963.229487:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203963.229491:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203963.229498:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788203963.229505:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203963.229517:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543369216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203963.229529:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203963.229532:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203963.229538:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543369216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203963.229549:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203963.229555:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25659) 00000800:00000200:3.1:1788203963.229743:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203963.229760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203963.230001:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203963.230003:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788203963.230235:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203963.230237:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1788203964.060181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203964.060187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203964.060196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203964.060200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203964.060207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203964.060211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203964.060218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203964.060221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203964.060278:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203964.060283:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203964.060286:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203964.572211:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320323333, at 4320323392 00000100:00000010:2.0:1788203964.572220:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcae0. 00010000:00000001:0.0:1788203965.084197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203965.084204:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203965.084207:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203965.084213:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203965.084221:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203965.084227:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203965.084234:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203965.084242:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203965.084250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203965.084262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203965.084271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203965.084272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203965.084274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203965.084276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203965.084327:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203965.084331:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203965.084333:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203966.108224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203966.108231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203966.108244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203966.108249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203966.108258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203966.108264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203966.108275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203966.108287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203966.108356:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203966.108359:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203966.108363:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203967.132222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203967.132227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203967.132235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203967.132238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203967.132245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203967.132248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203967.132255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203967.132257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203967.132326:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203967.132331:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203967.132336:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203968.156229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203968.156236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203968.156249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203968.156264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203968.156272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203968.156277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203968.156284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203968.156286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203968.156362:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203968.156368:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203968.156372:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788203968.348509:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203968.348517:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203968.348523:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203968.348536:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.348558:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203968.348562:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.348571:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203968.348576:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203968.348582:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203968.348593:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543369344:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203968.348612:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203968.348620:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203968.348624:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.348627:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203968.348635:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203968.348640:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203968.348648:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203968.348655:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203968.348676:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203968.348685:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543369344, portal 25 00000100:00000001:0.0:1788203968.348687:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203968.348688:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:0.0:1788203968.348694:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.348701:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203968.348703:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203968.348705:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203968.348708:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543369344, offset 0 00000400:00000200:0.0:1788203968.348714:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203968.348723:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.348729:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203968.348734:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203968.348738:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203968.348742:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203968.348745:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203968.348749:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470234:470234:256:4294967295] 192.168.201.118@tcp LPNI seq info [470093:470093:8:4294967295] 00000400:00000200:0.0:1788203968.348753:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.348766:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203968.348779:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.348788:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203968.348800:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203968.348815:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203968.348827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203968.348832:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788203968.348835:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.348838:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203968.348848:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203968.348856:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203968.348864:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203968.348867:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203968.348874:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203968.348881:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.348899:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.348905:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.348922:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.348925:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203968.348931:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203968.348938:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203968.348943:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320328453 00000100:00000010:0.0:1788203968.348951:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa060. 00000100:00000001:0.0:1788203968.348960:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788203968.348968:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320328453 00000100:00000010:0.0:1788203968.348974:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa900. 00000100:00000001:0.0:1788203968.348980:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788203968.348984:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203968.348985:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.348988:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203968.348995:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203968/real 1788203968] req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203968.349013:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203968.349015:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203968.349019:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203968.349024:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203968.349028:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203968.349030:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203968.349037:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203968.349057:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349074:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203968.349078:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349084:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.349123:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203968.349133:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349155:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203968.349165:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349188:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203968.349196:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349201:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.349206:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349213:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203968.349219:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203968.349224:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203968.349234:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203968.349240:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203968.349250:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203968.349266:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543369344:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203968.349275:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349279:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.349285:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543369344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349296:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203968.349300:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203968.349304:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203968.349307:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203968.349312:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203968.349315:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788203968.349320:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203968.349324:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:0.0:1788203968.349329:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203968.349332:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203968.349335:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:0.0:1788203968.349339:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203968.349343:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349348:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349357:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203968.349361:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.349366:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203968.349369:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203968.349372:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203968.349378:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349392:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203968.349395:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.349400:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203968.349403:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203968.349407:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203968.349413:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543369472:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203968.349417:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203968.349420:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203968.349423:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.349427:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203968.349431:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203968.349436:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203968.349445:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203968.349450:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203968.349456:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203968.349461:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543369472, portal 10 00000100:00000001:0.0:1788203968.349465:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203968.349467:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:0.0:1788203968.349479:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349486:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203968.349492:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203968.349497:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203968.349501:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543369472, offset 0 00000400:00000200:0.0:1788203968.349508:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203968.349513:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.349519:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203968.349525:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203968.349531:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203968.349535:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203968.349540:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203968.349546:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470235:470235:256:4294967295] 192.168.201.118@tcp LPNI seq info [470094:470094:8:4294967295] 00000400:00000200:0.0:1788203968.349553:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.349565:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203968.349576:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.349582:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203968.349588:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203968.349592:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203968.349599:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203968.349603:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788203968.349606:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.349608:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203968.349611:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203968.349615:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203968.349618:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203968.349620:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203968.349622:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203968.349626:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349632:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.349635:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349639:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.349640:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203968.349641:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203968.349645:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788203968.349647:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203968.349648:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320327187 00000100:00000001:0.0:1788203968.349650:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320328453 00000100:00000001:0.0:1788203968.349651:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320327187 00000100:00000001:0.0:1788203968.349653:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320328453 00000100:00000001:0.0:1788203968.349654:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203968.349656:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.349671:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203968.349674:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203968/real 1788203968] req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203968.349681:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203968.349682:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203968.349683:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203968.349685:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203968.349686:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203968.349688:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203968.349690:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203968.349696:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349700:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203968.349701:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349702:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.349703:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203968.349706:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349709:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203968.349712:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349717:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203968.349719:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349720:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.349721:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349722:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203968.349724:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203968.349725:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203968.349727:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203968.349729:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203968.349731:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203968.349736:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543369472:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203968.349739:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203968.349742:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349754:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203968.349755:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.349758:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203968.349759:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203968.349762:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203968.349765:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543369600:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203968.349769:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203968.349771:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203968.349772:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.349774:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203968.349778:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788203968.349780:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203968.349784:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203968.349787:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203968.349791:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203968.349794:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543369600, portal 10 00000100:00000001:0.0:1788203968.349796:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203968.349798:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:0.0:1788203968.349805:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349810:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203968.349812:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203968.349814:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203968.349816:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543369600, offset 0 00000400:00000200:0.0:1788203968.349819:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203968.349822:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.349824:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203968.349827:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203968.349829:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203968.349831:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203968.349833:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203968.349835:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470236:470236:256:4294967295] 192.168.201.118@tcp LPNI seq info [470095:470095:8:4294967295] 00000400:00000200:0.0:1788203968.349838:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.349843:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203968.349846:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.349849:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203968.349851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203968.349854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203968.349857:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203968.349859:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788203968.349861:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.349863:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203968.349865:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203968.349867:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203968.349869:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203968.349871:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203968.349872:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203968.349875:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349881:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.349883:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349888:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.349889:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203968.349891:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203968.349894:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:0.0:1788203968.349897:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203968.349899:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320327188 00000100:00000001:0.0:1788203968.349902:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320328453 00000100:00000001:0.0:1788203968.349904:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320327188 00000100:00000001:0.0:1788203968.349906:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320328453 00000100:00000001:0.0:1788203968.349909:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203968.349911:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.349913:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203968.349917:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203968/real 1788203968] req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203968.349927:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203968.349928:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203968.349931:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203968.349934:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203968.349936:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203968.349938:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203968.349942:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203968.349949:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349954:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203968.349955:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349956:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.349957:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203968.349959:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349964:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203968.349967:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349971:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203968.349972:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349973:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.349974:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.349975:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203968.349977:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203968.349978:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203968.349982:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203968.349983:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203968.349986:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203968.349991:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543369600:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203968.349993:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203968.349995:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.349999:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203968.350000:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.350002:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203968.350003:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203968.350005:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203968.350006:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543369728:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203968.350008:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203968.350010:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203968.350011:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.350012:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203968.350015:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203968.350016:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203968.350019:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203968.350020:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203968.350023:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203968.350025:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543369728, portal 4 00000100:00000001:0.0:1788203968.350027:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203968.350028:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:0.0:1788203968.350032:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350036:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203968.350038:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203968.350040:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203968.350042:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543369728, offset 0 00000400:00000200:0.0:1788203968.350044:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203968.350047:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.350049:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203968.350052:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203968.350053:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203968.350055:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203968.350057:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203968.350059:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470237:470237:256:4294967295] 192.168.201.118@tcp LPNI seq info [470096:470096:8:4294967295] 00000400:00000200:0.0:1788203968.350062:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.350066:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203968.350070:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.350072:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203968.350075:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203968.350077:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203968.350080:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203968.350082:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788203968.350084:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.350086:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203968.350088:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203968.350105:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203968.350108:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203968.350109:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203968.350111:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203968.350113:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350119:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.350121:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350126:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.350127:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203968.350130:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203968.350134:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:0.0:1788203968.350137:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203968.350139:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320327198 00000100:00000001:0.0:1788203968.350142:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320328453 00000100:00000001:0.0:1788203968.350144:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320327198 00000100:00000001:0.0:1788203968.350146:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320328453 00000100:00000001:0.0:1788203968.350148:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203968.350151:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.350153:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203968.350157:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203968/real 1788203968] req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203968.350165:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203968.350166:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203968.350168:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203968.350172:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203968.350173:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203968.350175:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203968.350178:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203968.350188:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350201:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203968.350203:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350205:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.350207:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203968.350212:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350218:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203968.350220:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350226:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203968.350227:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350229:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.350230:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350231:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203968.350232:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203968.350234:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203968.350236:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203968.350237:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203968.350240:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203968.350245:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543369728:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203968.350247:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350249:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.350251:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543369472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350255:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203968.350256:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203968.350257:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203968.350259:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203968.350260:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203968.350262:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203968.350263:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203968.350264:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:0.0:1788203968.350266:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203968.350268:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203968.350269:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:0.0:1788203968.350272:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203968.350273:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350275:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.350278:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543369600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350285:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203968.350287:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203968.350288:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788203968.350291:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203968.350292:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203968.350293:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203968.350296:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203968.350297:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:0.0:1788203968.350300:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203968.350301:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203968.350302:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:0.0:1788203968.350305:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203968.350306:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350308:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.350312:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543369728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350319:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203968.350320:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203968.350322:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203968.350324:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203968.350326:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203968.350327:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203968.350329:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203968.350331:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:0.0:1788203968.350333:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203968.350334:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203968.350335:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:0.0:1788203968.350338:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203968.350339:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350341:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350346:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203968.350347:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.350349:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203968.350352:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203968.350353:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203968.350357:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350365:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203968.350366:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.350368:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203968.350370:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203968.350371:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203968.350373:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543369856:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203968.350376:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203968.350377:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203968.350378:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203968.350380:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203968.350382:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203968.350384:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203968.350386:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203968.350388:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203968.350391:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203968.350393:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543369856, portal 4 00000100:00000001:0.0:1788203968.350394:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203968.350395:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:0.0:1788203968.350400:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350408:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203968.350410:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203968.350412:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203968.350415:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543369856, offset 0 00000400:00000200:0.0:1788203968.350419:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203968.350423:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.350427:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203968.350431:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203968.350434:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203968.350437:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203968.350440:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203968.350443:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470238:470238:256:4294967295] 192.168.201.118@tcp LPNI seq info [470097:470097:8:4294967295] 00000400:00000200:0.0:1788203968.350448:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203968.350454:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203968.350457:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.350460:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203968.350462:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203968.350464:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:0.1:1788203968.350467:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203968.350469:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:0.0:1788203968.350472:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203968.350473:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:0.0:1788203968.350476:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203968.350478:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203968.350480:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:0.0:1788203968.350481:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:0.0:1788203968.350483:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203968.350486:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350492:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.350494:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203993 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350498:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.350499:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203968.350502:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203968.350504:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:0.0:1788203968.350506:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203968.350507:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320327221 00000100:00000001:0.0:1788203968.350509:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320328453 00000100:00000001:0.0:1788203968.350511:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320327221 00000100:00000001:0.0:1788203968.350514:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320328453 00000100:00000001:0.0:1788203968.350516:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203968.350517:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.350519:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203968.350521:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203968/real 1788203968] req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203968.350526:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203968.350527:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203968.350528:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203968.350530:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000400:00000010:0.0:1788203968.350531:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000001:0.0:1788203968.350532:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203968.350534:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203968.350540:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350544:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203968.350545:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350546:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.350547:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203968.350549:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350553:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203968.350555:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350559:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203968.350561:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350562:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203968.350562:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350564:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203968.350566:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203968.350568:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203968.350570:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203968.350572:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203968.350574:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203968.350579:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543369856:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203968.350582:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350583:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203968.350585:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543369856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203993 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203968.350589:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203968.350590:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203968.350591:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203968.350592:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203968.350593:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203968.350594:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203968.350595:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203968.350596:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:0.0:1788203968.350598:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203968.350599:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203968.350600:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:0.0:1788203968.350601:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203968.350602:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350604:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203968.350606:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203968.350607:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203968.350609:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203968.350610:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203969.180188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203969.180194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203969.180201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203969.180204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203969.180209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203969.180212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203969.180220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203969.180224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203969.180312:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203969.180320:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203969.180325:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203969.692184:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320328453, at 4320328512 00000100:00000010:2.0:1788203969.692200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fa060. 00010000:00000001:0.0:1788203970.204187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203970.204193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203970.204211:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203970.204217:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203970.204224:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204233:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203970.204238:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204241:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204243:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203970.204249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203970.204251:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203970.204252:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204254:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203970.204256:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204257:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204260:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203970.204264:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203970.204265:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203970.204266:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204268:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203970.204270:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204272:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203970.204274:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203970.204327:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203970.204331:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203970.204333:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203971.228192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203971.228198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203971.228207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203971.228210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203971.228215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203971.228221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203971.228225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203971.228226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203971.228351:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203971.228354:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203971.228356:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203972.252187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203972.252202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203972.252211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203972.252217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203972.252226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203972.252230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203972.252236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203972.252239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203972.252334:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203972.252341:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203972.252346:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203973.276225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203973.276231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203973.276241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203973.276244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203973.276251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203973.276256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203973.276266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203973.276273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203973.276301:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203973.276311:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203973.276319:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788203973.468383:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203973.468390:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203973.468393:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203973.468403:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.468419:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203973.468422:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.468428:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203973.468432:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203973.468436:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203973.468442:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543369984:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203973.468448:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203973.468453:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203973.468457:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.468464:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203973.468473:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203973.468479:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203973.468491:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203973.468500:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036840. 00000400:00000010:0.0:1788203973.468512:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b770. 00000100:00000200:0.0:1788203973.468521:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543369984, portal 25 00000100:00000001:0.0:1788203973.468526:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203973.468528:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:0.0:1788203973.468540:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.468555:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203973.468562:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203973.468565:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b990. 00000100:00000200:0.0:1788203973.468583:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543369984, offset 0 00000400:00000200:0.0:1788203973.468593:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203973.468605:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.468613:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203973.468619:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203973.468624:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203973.468630:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203973.468635:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203973.468640:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470239:470239:256:4294967295] 192.168.201.118@tcp LPNI seq info [470098:470098:8:4294967295] 00000400:00000200:0.0:1788203973.468648:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.468666:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203973.468675:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203973.468682:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868af00. 00000800:00000200:0.0:1788203973.468696:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203973.468708:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788203973.468733:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203973.468744:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203973.468750:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320333573 00000100:00000010:0.0:1788203973.468758:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fac60. 00000100:00000001:0.0:1788203973.468766:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788203973.468776:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320333573 00000100:00000010:0.0:1788203973.468781:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa4e0. 00000100:00000001:0.0:1788203973.468788:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788203973.468795:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203973.468807:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.468815:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.468822:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203973.468831:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203973.468835:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788203973.468839:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203973.468843:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203973.468846:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203973.468852:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.468868:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203973.468872:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.468875:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203973.468878:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203973.468880:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203973.468885:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543370112:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203973.468891:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203973.468893:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203973.468897:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.468901:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203973.468907:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788203973.468912:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203973.468921:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203973.468927:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a930369f8. 00000400:00000010:0.0:1788203973.468935:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788203973.468941:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543370112, portal 10 00000100:00000001:0.0:1788203973.468947:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203973.468950:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:0.0:1788203973.468961:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.468976:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203973.468982:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203973.468986:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b198. 00000100:00000200:0.0:1788203973.468992:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543370112, offset 0 00000400:00000200:0.0:1788203973.469002:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203973.469010:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.469018:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203973.469026:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203973.469034:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203973.469041:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203973.469049:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203973.469057:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470240:470240:255:4294967295] 192.168.201.118@tcp LPNI seq info [470099:470099:7:4294967295] 00000400:00000200:0.0:1788203973.469069:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.469084:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203973.469118:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203973.469123:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a800. 00000800:00000200:0.0:1788203973.469127:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203973.469134:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788203973.469141:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203973.469147:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788203973.469156:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203973.469162:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320332307 00000100:00000001:0.0:1788203973.469168:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320333573 00000100:00000001:0.0:1788203973.469175:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320332307 00000100:00000001:0.0:1788203973.469179:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320333573 00000100:00000001:0.0:1788203973.469184:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203973.469187:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.469193:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203973.469200:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.469223:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203973.469228:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.469234:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203973.469239:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203973.469243:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203973.469251:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543370240:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203973.469260:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203973.469264:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203973.469267:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.469269:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203973.469274:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203973.469278:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203973.469282:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203973.469286:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788203973.469291:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788203973.469294:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543370240, portal 10 00000100:00000001:0.0:1788203973.469297:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203973.469298:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:0.0:1788203973.469305:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.469312:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203973.469315:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203973.469318:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b220. 00000100:00000200:0.0:1788203973.469320:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543370240, offset 0 00000400:00000200:0.0:1788203973.469324:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203973.469328:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.469332:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203973.469335:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203973.469339:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203973.469342:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203973.469345:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203973.469348:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470241:470241:254:4294967295] 192.168.201.118@tcp LPNI seq info [470100:470100:6:4294967295] 00000400:00000200:0.0:1788203973.469354:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.469362:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203973.469370:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203973.469374:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a300. 00000800:00000200:0.0:1788203973.469379:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203973.469384:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788203973.469387:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203973.469391:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:0.0:1788203973.469395:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203973.469398:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320332308 00000100:00000001:0.0:1788203973.469400:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320333573 00000100:00000001:0.0:1788203973.469402:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320332308 00000100:00000001:0.0:1788203973.469404:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320333573 00000100:00000001:0.0:1788203973.469407:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203973.469409:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.469411:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.469413:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203973.469417:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203973.469419:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788203973.469421:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203973.469422:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203973.469424:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203973.469428:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.469435:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203973.469436:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.469438:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203973.469440:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203973.469443:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203973.469446:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543370368:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203973.469449:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203973.469452:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203973.469453:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.469455:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203973.469460:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2bc00. 02000000:00000001:0.0:1788203973.469463:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203973.469467:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203973.469470:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036ec8. 00000400:00000010:0.0:1788203973.469474:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b088. 00000100:00000200:0.0:1788203973.469478:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543370368, portal 4 00000100:00000001:0.0:1788203973.469480:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203973.469481:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:0.0:1788203973.469488:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.469495:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203973.469498:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203973.469501:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44be58. 00000100:00000200:0.0:1788203973.469504:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543370368, offset 0 00000400:00000200:0.0:1788203973.469509:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203973.469514:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.469517:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203973.469521:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203973.469523:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203973.469525:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203973.469527:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203973.469531:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470242:470242:253:4294967295] 192.168.201.118@tcp LPNI seq info [470101:470101:5:4294967295] 00000400:00000200:0.0:1788203973.469535:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.469541:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203973.469545:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203973.469549:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868ac00. 00000800:00000200:0.0:1788203973.469553:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203973.469557:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788203973.469560:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203973.469563:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:0.0:1788203973.469566:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203973.469583:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320332317 00000100:00000001:0.0:1788203973.469587:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320333573 00000100:00000001:0.0:1788203973.469589:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320332317 00000100:00000001:0.0:1788203973.469591:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320333573 00000100:00000001:0.0:1788203973.469594:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203973.469596:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.469598:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203973.469604:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.469611:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203973.469613:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.469615:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203973.469617:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203973.469619:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203973.469622:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543370496:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203973.469626:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203973.469628:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203973.469630:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203973.469631:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203973.469636:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2a400. 02000000:00000001:0.0:1788203973.469638:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203973.469641:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203973.469644:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036630. 00000400:00000010:0.0:1788203973.469648:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44bb28. 00000100:00000200:0.0:1788203973.469651:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543370496, portal 4 00000100:00000001:0.0:1788203973.469653:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203973.469655:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:0.0:1788203973.469661:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203973.469668:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203973.469671:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203973.469673:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b3b8. 00000100:00000200:0.0:1788203973.469677:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543370496, offset 0 00000400:00000200:0.0:1788203973.469681:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203973.469686:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.469690:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203973.469693:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203973.469696:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203973.469698:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203973.469701:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203973.469704:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470243:470243:252:4294967295] 192.168.201.118@tcp LPNI seq info [470102:470102:4:4294967295] 00000400:00000200:0.0:1788203973.469709:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203973.469715:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203973.469720:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203973.469724:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868ae00. 00000800:00000200:0.0:1788203973.469728:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203973.469732:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788203973.469736:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203973.469739:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:0.0:1788203973.469742:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203973.469744:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320332340 00000100:00000001:0.0:1788203973.469746:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320333573 00000100:00000001:0.0:1788203973.469748:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320332340 00000100:00000001:0.0:1788203973.469750:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320333573 00000100:00000001:0.0:1788203973.469752:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203973.469754:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.469757:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.469759:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203973.469763:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203973.469765:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788203973.469767:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203973.469769:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203973.469770:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203973.469773:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203974.300206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203974.300224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203974.300245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203974.300248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203974.300265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203974.300270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203974.300275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203974.300289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203974.300384:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203974.300389:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203974.300393:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203974.812165:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320333573, at 4320333632 00000100:00000010:2.0:1788203974.812186:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fac60. 00010000:00000001:0.0:1788203975.324209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203975.324218:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203975.324224:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203975.324242:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203975.324251:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203975.324257:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203975.324264:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203975.324269:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203975.324276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203975.324278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203975.324282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203975.324283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203975.324287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203975.324288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203975.324347:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203975.324350:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203975.324353:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203976.348181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203976.348187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203976.348196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203976.348200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203976.348205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203976.348206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203976.348208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203976.348209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203976.348301:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203976.348305:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203976.348309:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788203976.540386:0:256123:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788203976.540425:0:256123:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788203976.540433:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203976.540437:0:256123:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788203976.540441:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788203976.540445:0:256123:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788203976.540451:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203976.540455:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868af00. 00000400:00000200:2.0:1788203976.540461:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203976.540467:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203976.540474:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b990 00000400:00000010:2.0:1788203976.540477:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b990. 00000100:00000001:2.0:1788203976.540484:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203976.540499:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203976.540539:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203976.540545:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203976.540554:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203976.540556:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203976.540567:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203976.540570:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a800. 00000400:00000200:2.0:1788203976.540574:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203976.540578:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203976.540582:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b198 00000400:00000010:2.0:1788203976.540584:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b198. 00000100:00000001:0.0:1788203976.540588:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788203976.540588:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788203976.540595:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000200:2.0:1788203976.540595:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203976.540607:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203976.540610:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.540616:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788203976.540617:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203976.540621:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203976.540628:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000100:00100000:0.0:1788203976.540630:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203973/real 1788203976] req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:2.0:1788203976.540633:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a300. 00000400:00000200:2.0:1788203976.540639:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203976.540648:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203976.540654:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b220 00000400:00000010:2.0:1788203976.540657:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b220. 00000400:00000001:0.0:1788203976.540660:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788203976.540660:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:0.0:1788203976.540665:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:2.0:1788203976.540668:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788203976.540671:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036840. 00000400:00000200:0.0:1788203976.540680:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b770 00000100:00000001:2.0:1788203976.540681:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000010:0.0:1788203976.540685:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b770. 00000100:00000040:2.0:1788203976.540687:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.540691:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788203976.540698:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1788203976.540700:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203976.540701:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788203976.540712:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203976.540717:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868ac00. 00000100:00000200:0.0:1788203976.540720:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788203976.540722:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203976.540732:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788203976.540739:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44be58 00000100:00000001:0.0:1788203976.540741:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000010:2.0:1788203976.540742:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44be58. 00000100:00000001:2.0:1788203976.540747:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788203976.540752:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.540759:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203976.540763:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788203976.540764:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.540768:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788203976.540770:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203976.540777:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788203976.540782:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203976.540787:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203976.540796:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000800:00000100:2.0:1788203976.540796:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788203976.540804:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868ae00. 00000100:00000040:0.0:1788203976.540808:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788203976.540810:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788203976.540820:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1788203976.540829:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1788203976.540829:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b3b8 00000400:00000010:2.0:1788203976.540835:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b3b8. 00000100:00000001:0.0:1788203976.540837:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203976.540840:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788203976.540842:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.540845:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1788203976.540849:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.540850:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203976.540855:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203976.540863:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788203976.540865:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788203976.540870:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788203998 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788203976.540874:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203976.540881:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788203976.540886:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203976.540889:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1788203976.540897:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203976.540909:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543369984:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203976.540919:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203976.540921:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203976.540926:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543369984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.540933:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203976.540938:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203976.540942:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:0.0:1788203976.540947:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203976.540950:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203976.540952:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788203976.540955:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203976.540958:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:0.0:1788203976.540962:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203976.540964:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203976.540966:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:0.0:1788203976.540971:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203976.540973:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.540976:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203976.540978:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203976.540980:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203976.540981:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203976.540986:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203973/real 1788203976] req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203976.540998:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203976.540999:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203976.541003:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036ec8. 00000400:00000200:0.0:1788203976.541006:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b088 00000400:00000010:0.0:1788203976.541008:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b088. 00000100:00000001:0.0:1788203976.541010:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203976.541014:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203976.541023:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541035:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203976.541037:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541039:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.541040:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203976.541044:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541051:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203976.541054:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541061:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203976.541064:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541065:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.541067:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541069:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203976.541073:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203976.541076:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203976.541081:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203976.541083:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203976.541087:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203976.541123:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543370368:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203976.541129:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203976.541134:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203973/real 1788203976] req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203976.541150:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203976.541152:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203976.541155:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036630. 00000400:00000200:0.0:1788203976.541158:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44bb28 00000400:00000010:0.0:1788203976.541160:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44bb28. 00000100:00000001:0.0:1788203976.541163:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203976.541167:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203976.541176:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541184:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203976.541186:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541188:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.541189:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203976.541213:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541220:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203976.541225:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541233:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203976.541235:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541237:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.541238:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541239:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203976.541242:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203976.541245:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203976.541249:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203976.541252:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203976.541256:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203976.541264:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543370496:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203976.541270:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203976.541274:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203973/real 1788203976] req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203976.541282:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203976.541283:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203976.541285:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a930369f8. 00000400:00000200:0.0:1788203976.541288:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788203976.541290:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788203976.541293:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203976.541296:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203976.541304:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541309:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203976.541311:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541312:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.541314:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203976.541317:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541324:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203976.541328:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541334:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203976.541336:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541338:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.541339:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541341:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203976.541343:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203976.541346:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203976.541350:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203976.541353:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203976.541358:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203976.541366:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543370112:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203976.541371:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203976.541375:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203973/real 1788203976] req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203976.541382:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203976.541384:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203976.541385:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788203976.541388:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788203976.541390:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788203976.541392:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203976.541395:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203976.541402:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541408:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203976.541410:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541411:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.541413:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203976.541416:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541423:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203976.541426:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541437:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203976.541439:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541441:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203976.541442:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541444:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203976.541447:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203976.541449:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203976.541453:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203976.541456:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203976.541460:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203976.541468:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543370240:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203976.541472:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541474:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203976.541483:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543370368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541490:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203976.541492:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203976.541494:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2bc00. 02000000:00000001:0.0:1788203976.541498:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203976.541500:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203976.541501:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203976.541504:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203976.541506:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:0.0:1788203976.541509:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203976.541510:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203976.541512:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:0.0:1788203976.541527:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203976.541529:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541531:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203976.541535:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543370496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541542:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203976.541544:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203976.541545:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2a400. 02000000:00000001:0.0:1788203976.541548:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203976.541550:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203976.541551:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203976.541553:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203976.541555:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:0.0:1788203976.541557:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203976.541559:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203976.541560:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:0.0:1788203976.541563:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203976.541564:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541565:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203976.541569:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543370112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541575:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203976.541577:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203976.541578:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788203976.541581:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203976.541582:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203976.541583:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203976.541585:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203976.541586:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:0.0:1788203976.541589:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203976.541590:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203976.541591:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:0.0:1788203976.541594:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203976.541595:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541597:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203976.541600:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543370240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788203998 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203976.541612:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203976.541613:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203976.541615:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203976.541617:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203976.541618:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203976.541619:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203976.541621:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203976.541623:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:0.0:1788203976.541625:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203976.541626:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203976.541628:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:0.0:1788203976.541630:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203976.541632:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541634:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203976.541642:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203976.541644:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203976.541647:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203976.541648:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203977.372151:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203977.372159:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203977.372166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203977.372169:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203977.372174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203977.372178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203977.372184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203977.372187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203977.372259:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203977.372264:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203977.372268:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203978.396286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203978.396296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203978.396304:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203978.396310:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203978.396321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203978.396330:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203978.396340:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203978.396346:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203978.396429:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203978.396435:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203978.396440:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203978.588247:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203978.588259:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203978.588263:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203978.588267:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203978.588272:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203978.588280:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25664 00000400:00000200:0.0:1788203978.588291:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203978.588298:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203978.588301:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203978.588304:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588307:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.588309:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588311:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203978.588315:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203978.588317:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203978.588320:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203978.588323:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203978.588325:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203978.588327:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203978.588330:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588331:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.588333:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588334:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203978.588336:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203978.588339:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588341:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203978.588343:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203978.588347:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203978.588350:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.588352:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203978.588354:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203978.588362:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788203978.588366:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203978.588370:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203978.588372:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.588375:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203978.588378:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203978.588383:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203978.588391:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788203978.588398:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203978.588407:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543370624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203978.588417:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203978.588422:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203978.588426:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543370624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203978.588461:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203978.588464:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203978.588467:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203978.588469:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203978.588470:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203978.588473:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203978.588476:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25664 00000400:00000200:0.0:1788203978.588496:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203978.588499:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203978.588501:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203978.588504:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588506:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.588509:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588510:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203978.588512:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203978.588513:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203978.588516:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203978.588519:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203978.588524:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203978.588547:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203978.588556:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588562:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.588568:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588578:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203978.588586:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203978.588594:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588599:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203978.588604:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203978.588615:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203978.588626:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.588631:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203978.588637:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203978.588645:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.588648:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203978.588654:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788203978.588658:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203978.588663:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203978.588666:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.588669:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203978.588671:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203978.588674:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203978.588690:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788203978.588698:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203978.588711:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543370752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203978.588743:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203978.588748:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203978.588758:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543370752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203978.588783:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203978.588789:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203978.588795:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203978.588797:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203978.588799:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203978.588806:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203978.588813:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25664 00000400:00000200:0.0:1788203978.588822:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203978.588837:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203978.588843:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203978.588851:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588858:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.588863:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588870:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203978.588875:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203978.588878:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203978.588882:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203978.588887:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203978.588891:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203978.588894:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203978.588899:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588906:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.588912:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588918:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203978.588921:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203978.588928:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203978.588933:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203978.588936:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203978.588941:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203978.588945:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.588947:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203978.588953:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203978.588960:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.588963:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203978.588970:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000800:00000200:3.1:1788203978.588976:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:0.0:1788203978.588977:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203978.588983:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203978.588986:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203978.588986:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:0.0:1788203978.588991:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203978.588995:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203978.588999:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203978.589007:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788203978.589015:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203978.589025:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543370880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203978.589041:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203978.589047:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203978.589056:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543370880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203978.589074:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203978.589082:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203978.589089:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788203978.589151:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203978.589156:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203978.589163:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203978.589170:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25664 00000400:00000200:0.0:1788203978.589176:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203978.589180:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203978.589183:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203978.589187:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589189:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.589191:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589193:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203978.589196:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203978.589198:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788203978.589200:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203978.589202:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203978.589205:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203978.589207:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203978.589211:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589213:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.589215:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589217:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203978.589220:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203978.589223:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589226:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203978.589229:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203978.589233:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203978.589235:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.589237:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203978.589240:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203978.589243:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.589245:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203978.589250:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788203978.589253:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203978.589256:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203978.589257:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.589260:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203978.589261:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203978.589263:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203978.589268:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788203978.589273:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203978.589278:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543371008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203978.589286:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203978.589289:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203978.589293:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543371008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203978.589302:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203978.589305:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203978.589310:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203978.589312:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203978.589314:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203978.589319:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203978.589322:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25664 00000400:00000200:0.0:1788203978.589331:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203978.589334:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203978.589339:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203978.589344:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589348:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.589353:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589357:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203978.589360:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203978.589362:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203978.589367:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203978.589371:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203978.589375:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203978.589380:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203978.589386:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589391:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203978.589396:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589401:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203978.589407:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203978.589415:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203978.589423:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203978.589429:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203978.589434:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203978.589440:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.589445:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203978.589448:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203978.589453:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.589458:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203978.589466:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788203978.589471:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203978.589477:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203978.589493:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203978.589495:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203978.589498:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203978.589501:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203978.589508:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788203978.589515:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203978.589527:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543371136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203978.589544:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203978.589549:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203978.589556:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543371136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203978.589588:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203978.589596:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25675) 00000800:00000200:3.1:1788203978.589902:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203978.589905:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203978.590245:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203978.590248:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203978.590573:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203978.590576:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788203978.590839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203978.590841:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788203979.420200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203979.420207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203979.420217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203979.420221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203979.420230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203979.420234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203979.420240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203979.420244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203979.420304:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203979.420308:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203979.420311:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203979.932183:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320338694, at 4320338752 00000100:00000010:2.0:1788203979.932200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc180. 00010000:00000001:0.0:1788203980.444175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203980.444180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203980.444190:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203980.444194:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203980.444199:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444204:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203980.444208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444211:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203980.444226:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203980.444229:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203980.444234:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444236:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203980.444238:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444239:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444241:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203980.444248:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203980.444249:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203980.444251:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444253:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203980.444254:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444256:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203980.444258:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203980.444319:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203980.444323:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203980.444326:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203981.468199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203981.468205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203981.468214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203981.468218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203981.468225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203981.468229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203981.468238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203981.468242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203981.468299:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203981.468304:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203981.468306:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203982.492185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203982.492191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203982.492199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203982.492204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203982.492208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203982.492212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203982.492218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203982.492221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203982.492287:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203982.492292:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203982.492296:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203983.516176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203983.516181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203983.516188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203983.516192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203983.516196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203983.516199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203983.516206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203983.516210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203983.516262:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203983.516266:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203983.516269:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203983.708207:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203983.708216:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203983.708219:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203983.708221:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203983.708225:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203983.708231:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25670 00000400:00000200:0.0:1788203983.708240:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203983.708245:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203983.708247:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203983.708250:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708252:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.708254:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708255:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203983.708258:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203983.708260:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203983.708262:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203983.708265:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203983.708267:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203983.708269:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203983.708271:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708272:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.708273:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708275:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203983.708277:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203983.708279:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708280:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203983.708283:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203983.708285:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203983.708287:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.708289:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203983.708291:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203983.708300:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788203983.708304:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203983.708308:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203983.708310:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.708313:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203983.708316:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203983.708321:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203983.708331:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788203983.708338:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203983.708347:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543371264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203983.708359:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203983.708363:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203983.708369:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543371264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203983.708515:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203983.708522:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203983.708538:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203983.708546:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203983.708548:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203983.708556:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203983.708563:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25670 00000400:00000200:0.0:1788203983.708571:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203983.708581:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203983.708587:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203983.708597:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708605:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.708611:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708618:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203983.708623:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203983.708627:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203983.708630:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203983.708635:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203983.708639:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203983.708643:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203983.708650:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708652:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.708654:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708659:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203983.708661:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203983.708664:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708669:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203983.708671:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203983.708678:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203983.708682:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788203983.708683:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788203983.708684:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:3.1:1788203983.708688:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000008:00100000:0.0:1788203983.708690:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203983.708696:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.708700:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203983.708704:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788203983.708708:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203983.708713:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203983.708716:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.708719:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203983.708722:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203983.708725:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203983.708729:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788203983.708735:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203983.708744:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543371392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203983.708754:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203983.708758:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203983.708766:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543371392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203983.708779:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203983.708784:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203983.708791:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203983.708794:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203983.708798:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203983.708805:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203983.708810:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25670 00000400:00000200:0.0:1788203983.708819:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203983.708828:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203983.708833:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203983.708839:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708843:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.708849:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708856:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203983.708863:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203983.708868:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203983.708873:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203983.708878:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203983.708885:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203983.708891:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203983.708900:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708905:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.708910:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708916:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203983.708923:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203983.708929:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203983.708935:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203983.708940:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203983.708944:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203983.708950:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.708953:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203983.708956:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203983.708959:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.708963:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203983.708969:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788203983.708973:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203983.708977:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203983.708980:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.708985:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203983.708987:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203983.708990:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203983.708996:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788203983.709002:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203983.709011:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543371520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203983.709051:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203983.709055:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203983.709063:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543371520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203983.709078:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203983.709084:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203983.709088:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788203983.709109:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203983.709111:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203983.709113:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203983.709116:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25670 00000400:00000200:0.0:1788203983.709120:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203983.709123:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203983.709124:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203983.709127:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709128:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.709130:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709133:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203983.709140:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203983.709142:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788203983.709146:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203983.709151:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203983.709157:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203983.709161:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203983.709169:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709175:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.709181:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709187:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203983.709191:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203983.709197:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709203:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203983.709209:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203983.709216:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203983.709222:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.709228:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203983.709233:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203983.709242:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.709247:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203983.709255:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788203983.709263:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203983.709268:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203983.709272:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.709277:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203983.709280:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203983.709285:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203983.709291:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788203983.709297:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203983.709306:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543371648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203983.709318:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203983.709322:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203983.709330:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543371648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203983.709344:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203983.709348:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203983.709353:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203983.709357:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203983.709360:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203983.709366:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203983.709372:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25670 00000400:00000200:0.0:1788203983.709380:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203983.709403:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203983.709408:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203983.709416:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709422:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.709429:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788203983.709430:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203983.709434:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1788203983.709436:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203983.709444:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203983.709449:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203983.709455:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203983.709462:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203983.709468:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203983.709473:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203983.709484:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709490:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203983.709498:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709504:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203983.709510:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203983.709515:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203983.709521:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203983.709525:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203983.709531:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203983.709534:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.709540:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203983.709543:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203983.709549:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.709554:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203983.709562:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788203983.709568:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203983.709576:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203983.709580:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203983.709584:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203983.709588:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203983.709592:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203983.709601:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788203983.709608:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203983.709619:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543371776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203983.709637:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203983.709641:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203983.709649:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543371776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203983.709666:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203983.709672:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25680) 00000800:00000200:3.1:1788203983.709819:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203983.709821:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203983.710058:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203983.710059:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788203983.710290:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203983.710292:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788203984.540177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203984.540183:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203984.540185:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203984.540190:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203984.540196:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203984.540200:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203984.540206:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203984.540210:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203984.540217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203984.540219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203984.540223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203984.540225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203984.540229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203984.540231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203984.540282:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203984.540285:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203984.540288:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203985.052205:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320343813, at 4320343872 00000100:00000010:2.0:1788203985.052214:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc720. 00010000:00000001:0.0:1788203985.564179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203985.564184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203985.564190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203985.564194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203985.564198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203985.564201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203985.564204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203985.564210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203985.564260:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203985.564263:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203985.564266:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203986.588180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203986.588185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203986.588191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203986.588195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203986.588201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203986.588205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203986.588209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203986.588213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203986.588283:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203986.588287:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203986.588291:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203987.612196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203987.612202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203987.612214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203987.612218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203987.612225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203987.612230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203987.612238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203987.612242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203987.612315:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203987.612319:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203987.612321:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203988.636188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203988.636192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203988.636199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203988.636203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203988.636207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203988.636209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203988.636214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203988.636217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203988.636303:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203988.636330:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203988.636333:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203988.828215:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203988.828230:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203988.828232:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203988.828237:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203988.828242:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203988.828249:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25675 00000400:00000200:0.0:1788203988.828260:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203988.828267:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203988.828271:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203988.828273:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828277:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.828278:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828280:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203988.828283:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203988.828285:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203988.828288:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203988.828290:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203988.828292:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203988.828294:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203988.828296:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828297:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.828299:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828300:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203988.828313:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203988.828316:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828317:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203988.828320:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203988.828323:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203988.828325:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828326:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203988.828328:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203988.828338:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788203988.828341:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203988.828344:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203988.828345:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828348:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203988.828350:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203988.828354:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203988.828360:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788203988.828366:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203988.828372:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543371904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203988.828385:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203988.828395:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203988.828401:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543371904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203988.828419:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203988.828421:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203988.828424:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203988.828425:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203988.828427:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203988.828429:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203988.828431:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25675 00000400:00000200:0.0:1788203988.828434:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203988.828437:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203988.828438:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203988.828441:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828442:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.828444:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828445:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203988.828448:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203988.828449:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203988.828450:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203988.828452:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203988.828454:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203988.828456:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203988.828458:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828460:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.828462:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828464:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203988.828467:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203988.828472:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828478:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203988.828481:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203988.828489:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203988.828496:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828501:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203988.828506:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203988.828513:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828518:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203988.828524:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788203988.828536:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203988.828540:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203988.828541:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828544:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203988.828547:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203988.828552:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203988.828557:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788203988.828564:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203988.828576:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543372032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203988.828588:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203988.828592:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203988.828599:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543372032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203988.828623:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203988.828629:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203988.828636:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203988.828641:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203988.828645:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203988.828650:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203988.828658:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25675 00000400:00000200:0.0:1788203988.828666:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203988.828669:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203988.828672:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203988.828675:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828677:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.828679:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828681:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203988.828686:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203988.828688:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203988.828693:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203988.828697:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203988.828702:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203988.828708:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203988.828716:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828721:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.828724:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828730:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203988.828733:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203988.828739:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828744:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203988.828746:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203988.828751:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203988.828757:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828759:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203988.828764:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203988.828770:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828774:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203988.828780:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788203988.828786:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203988.828792:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203988.828797:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828801:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203988.828805:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203988.828809:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203988.828816:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788203988.828822:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203988.828832:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543372160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203988.828846:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203988.828850:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203988.828857:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543372160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203988.828872:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203988.828879:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203988.828886:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788203988.828888:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203988.828890:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203988.828893:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203988.828897:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25675 00000400:00000200:0.0:1788203988.828909:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.1:1788203988.828911:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:0.0:1788203988.828916:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:3.1:1788203988.828919:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00080000:0.0:1788203988.828920:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203988.828926:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828929:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.828933:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828938:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203988.828940:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203988.828942:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788203988.828945:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203988.828947:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203988.828948:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203988.828951:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203988.828957:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828962:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.828966:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828969:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203988.828971:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203988.828974:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203988.828979:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203988.828982:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203988.828986:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203988.828988:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.828991:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203988.828993:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203988.828996:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.829000:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203988.829007:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788203988.829011:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203988.829016:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203988.829018:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.829021:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203988.829024:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203988.829026:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203988.829031:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788203988.829036:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203988.829045:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543372288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203988.829074:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203988.829076:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203988.829083:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543372288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203988.829111:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203988.829114:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203988.829116:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203988.829118:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203988.829119:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203988.829121:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203988.829124:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25675 00000400:00000200:0.0:1788203988.829127:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203988.829131:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203988.829137:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203988.829144:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.829150:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.829154:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.829157:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203988.829162:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203988.829164:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203988.829166:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203988.829171:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203988.829175:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203988.829178:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203988.829186:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.829190:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203988.829195:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203988.829198:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203988.829203:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203988.829210:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203988.829216:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203988.829221:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203988.829226:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203988.829233:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.829238:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203988.829243:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203988.829249:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.829254:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203988.829261:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788203988.829267:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203988.829274:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203988.829277:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203988.829282:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203988.829285:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203988.829289:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203988.829298:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788203988.829319:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203988.829334:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543372416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203988.829351:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203988.829357:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203988.829367:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543372416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203988.829382:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203988.829391:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25685) 00000800:00000200:3.1:1788203988.829635:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203988.829638:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203988.829954:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203988.829957:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203988.830295:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203988.830298:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788203988.830621:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203988.830624:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1788203989.660205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203989.660209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203989.660220:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203989.660223:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203989.660227:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660233:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203989.660238:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660244:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203989.660257:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203989.660259:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203989.660261:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660262:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203989.660264:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660266:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660267:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203989.660273:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203989.660274:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203989.660276:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660277:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203989.660278:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660279:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203989.660280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203989.660337:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203989.660341:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203989.660343:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788203990.172180:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320348934, at 4320348992 00000100:00000010:2.0:1788203990.172196:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc0c0. 00010000:00000001:0.0:1788203990.684167:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203990.684172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203990.684182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203990.684185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203990.684196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203990.684200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203990.684210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203990.684214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203990.684301:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203990.684306:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203990.684309:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203991.708191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203991.708198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203991.708209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203991.708214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203991.708232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203991.708237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203991.708248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203991.708267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203991.708324:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203991.708327:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203991.708330:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203992.732164:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203992.732169:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203992.732177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203992.732180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203992.732185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203992.732189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203992.732198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203992.732202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203992.732263:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203992.732267:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203992.732270:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788203993.756187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203993.756192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203993.756196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203993.756200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203993.756207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203993.756212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203993.756228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203993.756231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788203993.756302:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788203993.756306:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788203993.756309:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788203993.948191:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203993.948200:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788203993.948202:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203993.948205:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203993.948208:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203993.948238:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25680 00000400:00000200:0.0:1788203993.948247:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203993.948252:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203993.948255:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203993.948257:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948260:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.948261:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948264:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203993.948267:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203993.948268:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788203993.948271:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203993.948274:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788203993.948275:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788203993.948277:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788203993.948279:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948281:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.948282:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948283:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203993.948285:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788203993.948288:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948289:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203993.948292:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203993.948295:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203993.948297:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948299:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788203993.948301:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203993.948318:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788203993.948321:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788203993.948324:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203993.948325:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948327:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203993.948329:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203993.948333:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203993.948339:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788203993.948344:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203993.948352:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543372544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203993.948364:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203993.948368:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203993.948373:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543372544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203993.948401:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203993.948404:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203993.948407:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788203993.948408:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203993.948409:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203993.948412:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203993.948414:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25680 00000400:00000200:0.0:1788203993.948417:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203993.948420:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203993.948421:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203993.948423:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948425:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.948427:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948429:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203993.948431:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203993.948432:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788203993.948433:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203993.948436:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788203993.948437:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788203993.948439:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788203993.948441:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948442:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.948444:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948446:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203993.948450:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788203993.948454:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948458:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203993.948461:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203993.948466:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203993.948471:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948476:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203993.948482:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203993.948490:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948494:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203993.948502:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788203993.948508:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203993.948513:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203993.948517:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948523:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203993.948525:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203993.948529:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203993.948537:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788203993.948544:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203993.948554:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543372672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203993.948568:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203993.948574:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203993.948582:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543372672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203993.948593:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203993.948599:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203993.948605:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788203993.948609:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203993.948612:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203993.948617:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203993.948622:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25680 00000400:00000200:0.0:1788203993.948627:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203993.948631:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203993.948633:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203993.948638:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948643:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.948645:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948647:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203993.948650:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203993.948653:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788203993.948656:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203993.948659:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788203993.948661:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788203993.948664:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788203993.948670:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948673:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.948677:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948680:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203993.948683:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788203993.948689:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948695:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203993.948697:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203993.948700:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203993.948705:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948708:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203993.948712:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788203993.948715:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948717:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203993.948724:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788203993.948726:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788203993.948730:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203993.948731:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948735:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203993.948737:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203993.948741:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203993.948745:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788203993.948749:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203993.948753:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543372800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788203993.948757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203993.948764:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00000001:0.0:1788203993.948767:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203993.948770:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203993.948776:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543372800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203993.948793:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203993.948798:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203993.948807:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788203993.948810:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203993.948814:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203993.948818:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203993.948826:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25680 00000400:00000200:0.0:1788203993.948835:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203993.948843:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203993.948845:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203993.948850:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948854:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.948859:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948863:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203993.948867:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203993.948869:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788203993.948872:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203993.948875:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788203993.948877:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788203993.948879:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788203993.948882:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948887:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.948890:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948893:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203993.948896:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788203993.948901:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203993.948904:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203993.948908:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203993.948912:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203993.948914:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948917:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203993.948920:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788203993.948922:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948926:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203993.948934:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788203993.948937:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203993.948941:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203993.948944:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.948947:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203993.948950:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203993.948952:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203993.948957:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788203993.948962:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203993.948969:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543372928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203993.948981:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203993.948986:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203993.948992:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543372928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203993.949016:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788203993.949023:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788203993.949033:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788203993.949038:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788203993.949041:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788203993.949044:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788203993.949050:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25680 00000400:00000200:0.0:1788203993.949059:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203993.949066:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788203993.949069:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788203993.949075:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.949079:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.949083:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.949087:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788203993.949109:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788203993.949110:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788203993.949112:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788203993.949114:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788203993.949115:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788203993.949117:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788203993.949123:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.949129:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788203993.949133:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788203993.949137:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788203993.949140:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788203993.949144:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788203993.949147:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203993.949151:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788203993.949154:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203993.949159:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.949163:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788203993.949167:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788203993.949172:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.949176:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788203993.949180:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788203993.949185:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788203993.949189:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788203993.949192:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203993.949195:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788203993.949198:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788203993.949202:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788203993.949206:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788203993.949226:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203993.949234:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543373056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788203993.949244:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788203993.949248:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203993.949253:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543373056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203993.949268:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203993.949275:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25690) 00000800:00000200:3.1:1788203993.949420:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203993.949422:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203993.949642:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203993.949644:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203993.949845:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203993.949848:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788203993.950085:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788203993.950087:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1788203994.780177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203994.780183:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203994.780186:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203994.780190:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203994.780194:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203994.780197:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203994.780213:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203994.780215:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203994.780215:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203994.780223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203994.780224:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203994.780227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203994.780229:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203994.780233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203994.780235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203994.780236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203994.780242:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203994.780243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203994.780246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203994.780249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203994.780253:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203994.780256:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203994.780260:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788203995.292176:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320354053, at 4320354112 00000100:00000010:2.0:1788203995.292210:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fce40. 00000100:00000001:2.0:1788203995.804194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203995.804197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203995.804202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203995.804204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203995.804209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203995.804212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203995.804215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203995.804217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203995.804220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788203995.804224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203995.804226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203995.804230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203995.804231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203995.804233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203995.804240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203995.804241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203995.804246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203996.828202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203996.828207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203996.828210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203996.828211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203996.828214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788203996.828219:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203996.828220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203996.828223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203996.828225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203996.828226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203996.828229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203996.828230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203996.828233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203996.828236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203996.828237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203996.828240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203996.828244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203997.852199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203997.852200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203997.852206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203997.852209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203997.852212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203997.852214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203997.852214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203997.852222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203997.852223:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203997.852224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203997.852232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203997.852233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203997.852237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203997.852239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203997.852247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203997.852254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203997.852262:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203998.876206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203998.876209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203998.876214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203998.876215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788203998.876220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203998.876222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203998.876224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203998.876229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788203998.876229:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203998.876231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203998.876235:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203998.876237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203998.876240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203998.876240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203998.876247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788203998.876251:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788203998.876255:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.068541:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203999.068548:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203999.068553:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203999.068565:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.068584:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203999.068588:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.068596:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203999.068601:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203999.068609:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203999.068619:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543373184:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203999.068628:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203999.068634:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203999.068639:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.068643:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203999.068651:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2bc00. 02000000:00000001:0.0:1788203999.068658:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203999.068667:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203999.068676:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788203999.068686:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788203999.068697:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543373184, portal 25 00000100:00000001:0.0:1788203999.068702:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203999.068706:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:0.0:1788203999.068716:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.068754:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203999.068761:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203999.068766:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788203999.068774:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543373184, offset 0 00000400:00000200:0.0:1788203999.068788:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203999.068799:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.068810:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203999.068818:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203999.068826:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203999.068831:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203999.068838:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203999.068846:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470264:470264:256:4294967295] 192.168.201.118@tcp LPNI seq info [470123:470123:8:4294967295] 00000400:00000200:0.0:1788203999.068858:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.068881:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203999.068894:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.068906:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788203999.068919:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203999.068924:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203999.068936:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203999.068945:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788203999.068955:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.068961:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788203999.068970:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203999.068979:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203999.068990:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788203999.068995:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788203999.069003:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203999.069013:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069037:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.069047:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069137:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.069143:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203999.069154:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203999.069165:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203999.069173:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320359174 00000100:00000010:0.0:1788203999.069183:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fad20. 00000100:00000001:0.0:1788203999.069194:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788203999.069205:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320359174 00000100:00000010:0.0:1788203999.069215:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa7e0. 00000100:00000001:0.0:1788203999.069223:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788203999.069230:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203999.069235:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.069242:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203999.069260:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203999/real 1788203999] req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203999.069294:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203999.069298:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203999.069303:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788203999.069312:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788203999.069318:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788203999.069326:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203999.069336:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203999.069363:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069381:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203999.069387:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069392:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.069396:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203999.069409:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069422:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203999.069429:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069437:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203999.069441:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069443:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.069444:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069447:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203999.069449:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203999.069452:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203999.069455:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203999.069458:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203999.069462:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203999.069471:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543373184:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788203999.069476:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069479:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.069482:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543373184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069489:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203999.069491:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203999.069494:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2bc00. 02000000:00000001:0.0:1788203999.069497:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203999.069499:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203999.069501:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788203999.069503:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203999.069506:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:0.0:1788203999.069510:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203999.069512:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203999.069513:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:0.0:1788203999.069517:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203999.069518:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069521:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069529:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203999.069531:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.069533:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203999.069535:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788203999.069536:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203999.069540:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069548:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203999.069549:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.069552:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203999.069554:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203999.069556:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203999.069559:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203999.069562:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203999.069564:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203999.069566:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.069568:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203999.069571:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2bc00. 02000000:00000001:0.0:1788203999.069573:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203999.069577:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203999.069581:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788203999.069585:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788203999.069588:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543373312, portal 10 00000100:00000001:0.0:1788203999.069590:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203999.069591:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:0.0:1788203999.069598:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069605:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203999.069608:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203999.069610:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788203999.069613:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543373312, offset 0 00000400:00000200:0.0:1788203999.069617:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203999.069622:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.069626:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203999.069629:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203999.069632:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203999.069635:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203999.069637:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203999.069641:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470265:470265:256:4294967295] 192.168.201.118@tcp LPNI seq info [470124:470124:8:4294967295] 00000400:00000200:0.0:1788203999.069645:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.069652:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203999.069657:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.069660:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788203999.069664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203999.069668:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203999.069672:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203999.069675:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788203999.069680:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.069682:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788203999.069685:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203999.069689:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203999.069692:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788203999.069694:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788203999.069697:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203999.069701:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069709:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.069713:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069720:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.069722:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203999.069725:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203999.069729:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788203999.069733:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203999.069735:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320357907 00000100:00000001:0.0:1788203999.069738:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320359174 00000100:00000001:0.0:1788203999.069740:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320357907 00000100:00000001:0.0:1788203999.069743:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320359174 00000100:00000001:0.0:1788203999.069745:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203999.069747:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.069749:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203999.069753:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203999/real 1788203999] req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203999.069762:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203999.069764:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203999.069766:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788203999.069769:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788203999.069770:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788203999.069773:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203999.069776:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203999.069785:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069792:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203999.069794:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069796:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.069797:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203999.069801:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069808:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203999.069812:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069820:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203999.069822:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069824:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.069825:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.069827:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203999.069830:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203999.069832:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203999.069836:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203999.069838:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203999.069843:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203999.069851:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203999.069855:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203999.069859:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069866:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203999.069868:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.069870:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203999.069872:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203999.069874:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203999.069878:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203999.069881:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203999.069883:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203999.069885:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.069887:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203999.069890:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2a400. 02000000:00000001:0.0:1788203999.069893:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203999.069896:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203999.069899:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788203999.069902:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788203999.069905:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543373440, portal 10 00000100:00000001:0.0:1788203999.069907:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203999.069909:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:0.0:1788203999.069915:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.069922:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203999.069925:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203999.069927:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788203999.069930:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543373440, offset 0 00000400:00000200:0.0:1788203999.069934:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203999.069938:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.069942:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203999.069945:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203999.069947:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203999.069950:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203999.069953:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203999.069956:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470266:470266:256:4294967295] 192.168.201.118@tcp LPNI seq info [470125:470125:8:4294967295] 00000400:00000200:0.0:1788203999.069961:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.069967:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203999.069972:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.069976:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788203999.069980:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203999.069983:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203999.069988:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203999.069991:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788203999.069995:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.069997:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788203999.070000:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203999.070003:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203999.070007:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788203999.070009:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788203999.070012:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203999.070015:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070024:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.070027:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070034:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.070036:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203999.070039:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203999.070042:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:0.0:1788203999.070046:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203999.070048:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320357908 00000100:00000001:0.0:1788203999.070051:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320359174 00000100:00000001:0.0:1788203999.070053:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320357908 00000100:00000001:0.0:1788203999.070056:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320359174 00000100:00000001:0.0:1788203999.070058:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203999.070060:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.070062:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203999.070066:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203999/real 1788203999] req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203999.070075:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203999.070077:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203999.070078:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788203999.070081:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788203999.070083:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788203999.070085:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203999.070089:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203999.070117:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070139:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203999.070141:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070143:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.070145:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203999.070149:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070156:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203999.070160:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070168:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203999.070171:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070172:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.070174:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070176:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203999.070178:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203999.070181:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203999.070185:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203999.070187:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203999.070191:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203999.070200:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788203999.070204:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203999.070208:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070215:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203999.070216:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.070219:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203999.070221:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203999.070224:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203999.070227:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203999.070230:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203999.070233:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203999.070234:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.070236:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203999.070240:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788203999.070242:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203999.070246:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203999.070248:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788203999.070253:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788203999.070255:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543373568, portal 4 00000100:00000001:0.0:1788203999.070258:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203999.070259:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:0.0:1788203999.070265:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070272:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203999.070275:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203999.070277:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788203999.070280:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543373568, offset 0 00000400:00000200:0.0:1788203999.070284:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203999.070288:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.070292:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203999.070295:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203999.070298:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203999.070301:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203999.070303:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203999.070307:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470267:470267:256:4294967295] 192.168.201.118@tcp LPNI seq info [470126:470126:8:4294967295] 00000400:00000200:0.0:1788203999.070311:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.070318:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203999.070324:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.070327:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788203999.070332:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203999.070335:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203999.070339:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203999.070342:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788203999.070347:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.070349:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788203999.070352:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203999.070356:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203999.070360:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788203999.070361:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788203999.070365:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203999.070368:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070377:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.070380:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070387:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.070389:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203999.070392:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203999.070396:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:0.0:1788203999.070399:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203999.070401:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320357918 00000100:00000001:0.0:1788203999.070404:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320359174 00000100:00000001:0.0:1788203999.070406:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320357918 00000100:00000001:0.0:1788203999.070408:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320359174 00000100:00000001:0.0:1788203999.070410:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203999.070412:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.070414:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203999.070419:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203999/real 1788203999] req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203999.070428:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203999.070429:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203999.070431:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788203999.070434:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788203999.070436:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788203999.070438:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203999.070442:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203999.070450:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070457:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203999.070458:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070460:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.070462:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203999.070466:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070473:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203999.070476:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070483:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203999.070485:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070487:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.070488:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070491:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203999.070493:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203999.070495:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203999.070499:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203999.070501:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203999.070506:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203999.070514:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203999.070518:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788203999.070521:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070528:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788203999.070529:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.070532:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788203999.070533:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788203999.070535:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788203999.070538:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203999.070542:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788203999.070544:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788203999.070545:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788203999.070547:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788203999.070551:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203999.070554:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788203999.070557:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788203999.070560:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788203999.070563:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788203999.070566:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543373696, portal 4 00000100:00000001:0.0:1788203999.070568:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788203999.070570:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:0.0:1788203999.070576:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070583:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788203999.070586:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788203999.070588:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788203999.070590:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543373696, offset 0 00000400:00000200:0.0:1788203999.070595:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788203999.070599:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.070603:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788203999.070606:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788203999.070609:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788203999.070611:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788203999.070614:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788203999.070617:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470268:470268:256:4294967295] 192.168.201.118@tcp LPNI seq info [470127:470127:8:4294967295] 00000400:00000200:0.0:1788203999.070621:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788203999.070628:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788203999.070633:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.070636:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788203999.070640:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788203999.070643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:0.1:1788203999.070647:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788203999.070650:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:0.0:1788203999.070654:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788203999.070656:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788203999.070659:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788203999.070662:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788203999.070665:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788203999.070667:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788203999.070670:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788203999.070674:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070681:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.070684:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070690:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.070692:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788203999.070695:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788203999.070698:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:0.0:1788203999.070701:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788203999.070704:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320357941 00000100:00000001:0.0:1788203999.070706:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320359174 00000100:00000001:0.0:1788203999.070708:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320357941 00000100:00000001:0.0:1788203999.070711:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320359174 00000100:00000001:0.0:1788203999.070713:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788203999.070715:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.070717:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788203999.070720:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788203999/real 1788203999] req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788203999.070728:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788203999.070729:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788203999.070731:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788203999.070734:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788203999.070736:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788203999.070738:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788203999.070742:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788203999.070750:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070757:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788203999.070759:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070760:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.070762:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788203999.070766:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070774:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788203999.070778:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070785:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788203999.070787:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070789:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788203999.070790:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070792:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788203999.070794:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788203999.070797:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788203999.070800:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788203999.070803:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788203999.070807:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788203999.070815:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788203999.070819:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070821:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.070825:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543373312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070832:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203999.070833:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203999.070836:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2bc00. 02000000:00000001:0.0:1788203999.070838:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203999.070840:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203999.070841:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203999.070843:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203999.070845:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:0.0:1788203999.070848:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203999.070850:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203999.070851:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:0.0:1788203999.070854:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203999.070855:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070857:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.070861:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543373440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070874:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203999.070875:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203999.070876:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2a400. 02000000:00000001:0.0:1788203999.070879:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203999.070880:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203999.070881:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788203999.070884:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203999.070885:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:0.0:1788203999.070887:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203999.070888:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203999.070890:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:0.0:1788203999.070893:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203999.070894:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070896:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.070900:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543373568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070906:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203999.070907:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203999.070909:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788203999.070911:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203999.070912:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203999.070913:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203999.070915:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203999.070916:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:0.0:1788203999.070919:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203999.070920:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203999.070921:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:0.0:1788203999.070924:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203999.070925:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070927:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788203999.070930:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543373696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788203999.070937:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788203999.070938:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788203999.070940:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788203999.070942:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788203999.070943:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788203999.070945:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788203999.070947:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788203999.070948:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:0.0:1788203999.070950:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788203999.070951:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788203999.070953:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:0.0:1788203999.070955:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788203999.070956:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070958:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788203999.070963:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788203999.070964:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788203999.070966:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788203999.070968:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203999.900221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788203999.900226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203999.900228:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788203999.900234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203999.900234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900238:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203999.900242:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788203999.900243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788203999.900246:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900249:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788203999.900249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788203999.900253:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900255:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203999.900259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203999.900261:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203999.900264:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788203999.900266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788203999.900266:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788203999.900269:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203999.900272:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203999.900275:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788203999.900275:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900280:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203999.900282:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900284:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900286:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788203999.900292:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788203999.900293:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788203999.900295:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900297:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788203999.900298:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900301:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788203999.900303:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204000.412201:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320359174, at 4320359232 00000100:00000010:2.0:1788204000.412216:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26fad20. 00000100:00000001:2.0:1788204000.924155:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204000.924162:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204000.924163:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204000.924169:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204000.924174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204000.924174:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204000.924178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204000.924179:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204000.924182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204000.924184:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204000.924187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204000.924189:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204000.924191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204000.924195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204000.924195:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204000.924199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204000.924201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204000.924222:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204000.924225:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204000.924227:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204001.948183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204001.948186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204001.948189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204001.948190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204001.948194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204001.948196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204001.948200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204001.948200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204001.948207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204001.948209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204001.948211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204001.948214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204001.948217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204001.948218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204001.948223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204001.948226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204001.948228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204002.972180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204002.972184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204002.972189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204002.972190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204002.972198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204002.972200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204002.972203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204002.972208:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204002.972210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204002.972215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204002.972215:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204002.972219:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204002.972221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204002.972225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204002.972225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204002.972229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204002.972234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204003.996170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204003.996174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204003.996177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204003.996178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204003.996180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204003.996185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204003.996188:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204003.996192:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204003.996195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204003.996196:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204003.996199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204003.996201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204003.996205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204003.996208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204003.996213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204003.996213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204003.996216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204003.996234:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204003.996237:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204003.996239:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204004.188412:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204004.188418:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204004.188422:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204004.188432:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.188447:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204004.188450:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.188456:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204004.188460:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204004.188465:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204004.188474:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543373824:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788204004.188480:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204004.188485:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204004.188489:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.188494:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204004.188501:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788204004.188504:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204004.188511:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788204004.188516:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788204004.188523:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788204004.188530:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543373824, portal 25 00000100:00000001:0.0:1788204004.188534:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204004.188536:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:0.0:1788204004.188542:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.188551:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204004.188560:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788204004.188565:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788204004.188571:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543373824, offset 0 00000400:00000200:0.0:1788204004.188579:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788204004.188591:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.188600:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788204004.188608:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204004.188615:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788204004.188622:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204004.188628:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788204004.188637:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470269:470269:256:4294967295] 192.168.201.118@tcp LPNI seq info [470128:470128:8:4294967295] 00000400:00000200:0.0:1788204004.188646:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.188663:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788204004.188674:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.188682:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788204004.188694:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788204004.188700:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204004.188708:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788204004.188712:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204004.188718:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.188720:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788204004.188726:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204004.188733:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204004.188739:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788204004.188742:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788204004.188746:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204004.188751:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.188763:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.188767:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.188774:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.188776:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204004.188781:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204004.188786:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788204004.188788:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320364293 00000100:00000010:0.0:1788204004.188794:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26faba0. 00000100:00000001:0.0:1788204004.188801:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788204004.188808:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320364293 00000100:00000010:0.0:1788204004.188812:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa300. 00000100:00000001:0.0:1788204004.188815:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788204004.188819:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204004.188823:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.188827:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204004.188833:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204004/real 1788204004] req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204004.188847:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204004.188850:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204004.188852:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788204004.188857:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788204004.188860:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788204004.188862:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204004.188867:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204004.188879:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.188889:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204004.188891:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.188893:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.188896:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204004.188902:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.188910:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204004.188916:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.188927:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204004.188930:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.188933:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.188936:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.188939:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204004.188941:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204004.188944:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204004.188951:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788204004.188955:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204004.188963:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204004.188974:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543373824:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788204004.188979:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.188983:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.188986:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543373824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.188995:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204004.188998:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204004.189001:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788204004.189003:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204004.189006:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788204004.189008:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788204004.189012:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788204004.189016:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:0.0:1788204004.189020:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204004.189024:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204004.189027:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:0.0:1788204004.189032:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204004.189046:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189050:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189059:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204004.189062:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.189066:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204004.189069:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204004.189072:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204004.189079:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189124:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204004.189126:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.189132:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204004.189133:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204004.189136:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204004.189141:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788204004.189147:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204004.189150:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204004.189151:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.189154:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204004.189196:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788204004.189201:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204004.189208:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788204004.189214:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788204004.189221:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788204004.189227:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543373952, portal 10 00000100:00000001:0.0:1788204004.189229:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204004.189232:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:0.0:1788204004.189242:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189253:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204004.189258:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788204004.189260:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788204004.189271:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543373952, offset 0 00000400:00000200:0.0:1788204004.189277:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788204004.189282:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.189287:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788204004.189291:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204004.189295:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788204004.189298:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204004.189302:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788204004.189306:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470270:470270:256:4294967295] 192.168.201.118@tcp LPNI seq info [470129:470129:8:4294967295] 00000400:00000200:0.0:1788204004.189310:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.189318:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788204004.189322:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.189326:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788204004.189332:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788204004.189336:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204004.189343:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788204004.189348:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204004.189354:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.189357:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788204004.189361:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204004.189366:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204004.189370:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788204004.189373:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788204004.189376:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204004.189380:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189386:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.189391:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189409:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.189413:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204004.189416:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204004.189419:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788204004.189423:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788204004.189424:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320363027 00000100:00000001:0.0:1788204004.189426:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320364293 00000100:00000001:0.0:1788204004.189428:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320363027 00000100:00000001:0.0:1788204004.189430:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320364293 00000100:00000001:0.0:1788204004.189432:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204004.189433:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.189434:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204004.189437:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204004/real 1788204004] req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204004.189442:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204004.189442:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204004.189443:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788204004.189445:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788204004.189446:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788204004.189447:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204004.189449:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204004.189453:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189456:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204004.189457:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189458:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.189459:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204004.189461:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189464:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204004.189466:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189471:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204004.189472:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189473:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.189474:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189475:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204004.189476:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204004.189478:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204004.189480:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788204004.189481:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204004.189484:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204004.189488:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543373952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788204004.189490:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204004.189491:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189495:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204004.189496:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.189497:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204004.189498:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204004.189499:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204004.189501:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543374080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788204004.189503:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204004.189504:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204004.189505:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.189506:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204004.189508:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788204004.189509:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204004.189510:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788204004.189512:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788204004.189513:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788204004.189515:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543374080, portal 10 00000100:00000001:0.0:1788204004.189516:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204004.189517:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:0.0:1788204004.189520:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189527:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204004.189529:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788204004.189530:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788204004.189531:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543374080, offset 0 00000400:00000200:0.0:1788204004.189533:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788204004.189535:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.189537:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788204004.189539:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204004.189540:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788204004.189541:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204004.189543:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788204004.189545:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470271:470271:256:4294967295] 192.168.201.118@tcp LPNI seq info [470130:470130:8:4294967295] 00000400:00000200:0.0:1788204004.189547:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.189551:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788204004.189553:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.189555:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788204004.189558:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788204004.189559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204004.189561:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788204004.189563:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204004.189565:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.189566:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788204004.189568:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204004.189570:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204004.189571:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788204004.189572:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788204004.189574:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204004.189576:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189580:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.189582:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189590:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.189592:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204004.189594:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204004.189597:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:0.0:1788204004.189600:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788204004.189601:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320363028 00000100:00000001:0.0:1788204004.189603:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320364293 00000100:00000001:0.0:1788204004.189605:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320363028 00000100:00000001:0.0:1788204004.189607:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320364293 00000100:00000001:0.0:1788204004.189609:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204004.189611:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.189613:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204004.189616:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204004/real 1788204004] req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204004.189624:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204004.189625:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204004.189627:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788204004.189629:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788204004.189631:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788204004.189633:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204004.189636:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204004.189642:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189646:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204004.189647:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189648:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.189649:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204004.189651:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189655:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204004.189658:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189664:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204004.189666:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189668:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.189669:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189670:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204004.189673:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204004.189675:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204004.189678:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788204004.189680:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204004.189685:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204004.189693:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543374080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:0.0:1788204004.189696:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189698:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.189701:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543373952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189705:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204004.189707:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204004.189708:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788204004.189709:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204004.189710:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788204004.189711:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788204004.189712:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788204004.189713:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:0.0:1788204004.189715:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204004.189716:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204004.189717:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:0.0:1788204004.189719:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204004.189720:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189721:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.189722:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543374080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189726:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204004.189727:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204004.189727:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788204004.189729:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204004.189729:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788204004.189730:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:0.0:1788204004.189731:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788204004.189731:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:0.0:1788204004.189733:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204004.189733:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204004.189734:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:0.0:1788204004.189735:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204004.189736:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189737:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189740:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204004.189741:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.189743:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204004.189744:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204004.189745:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204004.189748:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189754:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204004.189755:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.189757:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204004.189758:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204004.189760:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204004.189762:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543374208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788204004.189767:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204004.189769:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204004.189771:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.189772:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204004.189775:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788204004.189777:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204004.189780:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788204004.189783:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788204004.189786:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788204004.189789:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543374208, portal 4 00000100:00000001:0.0:1788204004.189791:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204004.189792:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:0.0:1788204004.189797:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189804:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204004.189806:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788204004.189808:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788204004.189811:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543374208, offset 0 00000400:00000200:0.0:1788204004.189814:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788204004.189819:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.189822:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788204004.189825:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204004.189827:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788204004.189829:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204004.189832:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788204004.189835:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470272:470272:256:4294967295] 192.168.201.118@tcp LPNI seq info [470131:470131:8:4294967295] 00000400:00000200:0.0:1788204004.189839:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.189845:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788204004.189850:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.189853:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788204004.189856:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788204004.189859:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204004.189861:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788204004.189863:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204004.189865:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.189866:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788204004.189868:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204004.189870:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204004.189872:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788204004.189873:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788204004.189875:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204004.189877:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189882:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.189884:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189888:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.189888:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204004.189890:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204004.189892:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:0.0:1788204004.189894:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788204004.189895:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320363037 00000100:00000001:0.0:1788204004.189897:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320364293 00000100:00000001:0.0:1788204004.189898:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320363037 00000100:00000001:0.0:1788204004.189900:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320364293 00000100:00000001:0.0:1788204004.189901:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204004.189902:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.189903:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204004.189905:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204004/real 1788204004] req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204004.189909:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204004.189910:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204004.189911:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788204004.189913:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788204004.189913:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788204004.189915:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204004.189916:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204004.189921:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189925:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204004.189926:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189927:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.189927:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204004.189929:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189933:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204004.189935:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189938:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204004.189939:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189940:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.189941:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.189942:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204004.189943:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204004.189944:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204004.189946:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788204004.189947:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204004.189949:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204004.189954:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543374208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788204004.189956:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204004.189958:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189962:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204004.189962:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.189964:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204004.189965:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204004.189966:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204004.189967:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543374336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788204004.189969:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204004.189970:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204004.189971:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204004.189972:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204004.189973:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788204004.189975:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204004.189976:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788204004.189977:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788204004.189979:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788204004.189981:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543374336, portal 4 00000100:00000001:0.0:1788204004.189982:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204004.189983:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:0.0:1788204004.189986:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.189990:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204004.189991:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788204004.189992:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788204004.189993:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543374336, offset 0 00000400:00000200:0.0:1788204004.189995:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788204004.189997:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.189999:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788204004.190000:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204004.190002:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788204004.190004:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204004.190005:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788204004.190007:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470273:470273:256:4294967295] 192.168.201.118@tcp LPNI seq info [470132:470132:8:4294967295] 00000400:00000200:0.0:1788204004.190010:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204004.190015:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788204004.190019:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.190021:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788204004.190024:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788204004.190027:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:0.1:1788204004.190030:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788204004.190033:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:0.0:1788204004.190050:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204004.190052:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788204004.190055:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204004.190058:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204004.190061:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788204004.190062:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788204004.190065:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204004.190068:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.190075:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.190078:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.190082:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.190083:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204004.190085:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204004.190087:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:0.0:1788204004.190102:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788204004.190103:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4320363061 00000100:00000001:0.0:1788204004.190105:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4320364293 00000100:00000001:0.0:1788204004.190106:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4320363061 00000100:00000001:0.0:1788204004.190108:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4320364293 00000100:00000001:0.0:1788204004.190109:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204004.190111:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.190112:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204004.190114:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204004/real 1788204004] req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204004.190119:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204004.190120:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204004.190121:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788204004.190122:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788204004.190123:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788204004.190125:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204004.190126:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204004.190131:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.190134:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204004.190135:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.190136:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.190137:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204004.190139:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.190142:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204004.190144:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.190148:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204004.190150:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.190151:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204004.190151:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.190153:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204004.190154:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204004.190155:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204004.190157:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788204004.190158:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204004.190161:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204004.190165:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543374336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:0.0:1788204004.190167:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.190168:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.190169:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543374208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.190174:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204004.190175:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204004.190176:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:0.0:1788204004.190178:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204004.190180:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788204004.190181:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788204004.190183:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788204004.190184:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:0.0:1788204004.190186:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204004.190187:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204004.190188:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:0.0:1788204004.190190:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204004.190191:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.190193:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204004.190196:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543374336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204004.190202:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204004.190203:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204004.190205:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788204004.190207:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204004.190208:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788204004.190209:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:0.0:1788204004.190212:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788204004.190213:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:0.0:1788204004.190215:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204004.190216:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204004.190217:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:0.0:1788204004.190219:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204004.190221:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.190222:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204004.190225:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204004.190226:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204004.190227:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204004.190228:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204005.020186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204005.020191:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204005.020194:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204005.020198:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204005.020204:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204005.020207:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204005.020209:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204005.020212:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204005.020216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204005.020217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204005.020220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204005.020222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204005.020224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204005.020225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204005.532173:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320364293, at 4320364352 00000100:00000010:2.0:1788204005.532187:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faba0. 00010000:00000001:0.0:1788204006.044201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204006.044206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204006.044214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204006.044217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204006.044222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204006.044225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204006.044229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204006.044232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204006.044293:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204006.044298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204006.044301:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204007.068190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204007.068194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204007.068203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204007.068207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204007.068215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204007.068219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204007.068227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204007.068230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204007.068261:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204007.068267:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204007.068270:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204008.092171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204008.092176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204008.092183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204008.092186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204008.092191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204008.092193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204008.092198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204008.092202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204008.092262:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204008.092266:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204008.092269:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204009.116180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204009.116184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204009.116191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204009.116195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204009.116198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204009.116202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204009.116205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204009.116208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204009.116262:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204009.116266:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204009.116269:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204009.308441:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204009.308446:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204009.308449:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1788204009.308458:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204009.308471:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204009.308474:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204009.308478:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788204009.308481:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204009.308485:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204009.308490:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543374464:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788204009.308493:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788204009.308498:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788204009.308501:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204009.308504:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1788204009.308510:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788204009.308513:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204009.308520:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:0.0:1788204009.308525:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7a93036318. 00000400:00000010:0.0:1788204009.308532:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b4c8. 00000100:00000200:0.0:1788204009.308539:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543374464, portal 25 00000100:00000001:0.0:1788204009.308542:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204009.308545:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:0.0:1788204009.308550:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204009.308560:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788204009.308563:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:0.0:1788204009.308567:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b9f44b330. 00000100:00000200:0.0:1788204009.308572:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543374464, offset 0 00000400:00000200:0.0:1788204009.308582:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788204009.308591:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204009.308599:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:0.0:1788204009.308606:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788204009.308611:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:0.0:1788204009.308615:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788204009.308620:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788204009.308626:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [470274:470274:256:4294967295] 192.168.201.118@tcp LPNI seq info [470133:470133:8:4294967295] 00000400:00000200:0.0:1788204009.308632:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204009.308648:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:0.0:1788204009.308656:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204009.308663:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7aa868a400. 00000800:00000200:0.0:1788204009.308670:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.0:1788204009.308677:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:0.1:1788204009.308685:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:0.1:1788204009.308690:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000100:0.0:1788204009.308696:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:0.0:1788204009.308699:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7aa868a400. 00000400:00000200:0.0:1788204009.308705:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1788204009.308711:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1788204009.308719:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b330 00000400:00000010:0.0:1788204009.308722:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b330. 00000100:00000001:0.0:1788204009.308730:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204009.308737:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204009.308750:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204009.308755:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204034 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204009.308761:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204009.308764:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788204009.308770:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788204009.308776:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788204009.308780:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4320369413 00000100:00000010:0.0:1788204009.308785:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26faf00. 00000100:00000001:0.0:1788204009.308792:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788204009.308798:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4320369413 00000100:00000010:0.0:1788204009.308804:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab26fa300. 00000100:00000001:0.0:1788204009.308808:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1788204009.308812:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204009.308814:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204009.308816:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788204009.308824:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204009/real 1788204009] req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788204009.308843:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788204009.308845:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1788204009.308848:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7a93036318. 00000400:00000200:0.0:1788204009.308853:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b9f44b4c8 00000400:00000010:0.0:1788204009.308855:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b9f44b4c8. 00000100:00000001:0.0:1788204009.308858:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1788204009.308864:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788204009.308875:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204009.308900:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788204009.308902:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204009.308908:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204009.308909:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788204009.308914:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204009.308923:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204009.308930:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204034 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204009.308939:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788204009.308943:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204009.308944:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204009.308962:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204009.308965:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204009.308968:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788204009.308972:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788204009.308979:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:0.0:1788204009.308983:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204009.308988:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204034 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788204009.309005:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543374464:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788204009.309015:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204009.309018:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204009.309023:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543374464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204034 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204009.309032:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788204009.309035:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788204009.309039:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:0.0:1788204009.309041:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788204009.309044:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:0.0:1788204009.309047:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:0.0:1788204009.309051:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:0.0:1788204009.309054:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:0.0:1788204009.309058:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788204009.309061:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1788204009.309064:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:0.0:1788204009.309069:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788204009.309071:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204009.309073:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204009.309081:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204009.309084:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204009.309088:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788204009.309127:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000800:00000200:1.1:1788204009.309377:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204009.309380:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788204009.309656:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204009.309658:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788204009.309907:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204009.309909:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788204009.310089:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204009.310103:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00010000:00000001:0.0:1788204010.140273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204010.140275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204010.140279:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204010.140281:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204010.140283:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140286:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204010.140287:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140288:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140290:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204010.140292:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204010.140293:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204010.140294:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140295:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204010.140296:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140297:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140298:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204010.140302:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204010.140302:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204010.140303:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140304:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204010.140304:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140305:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204010.140306:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204010.140320:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204010.140321:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204010.140323:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204010.652172:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320369413, at 4320369472 00000100:00000010:2.0:1788204010.652189:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab26faf00. 00010000:00000001:0.0:1788204011.164185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204011.164189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204011.164198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204011.164201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204011.164204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204011.164207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204011.164211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204011.164214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204011.164272:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204011.164276:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204011.164279:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204012.188170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204012.188176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204012.188186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204012.188189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204012.188195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204012.188198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204012.188201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204012.188203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204012.188259:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204012.188263:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204012.188267:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204013.212198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204013.212211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204013.212218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204013.212222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204013.212228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204013.212232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204013.212240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204013.212242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204013.212269:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204013.212271:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204013.212273:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204014.236206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204014.236212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204014.236225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204014.236229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204014.236232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204014.236236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204014.236242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204014.236244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204014.236299:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204014.236303:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204014.236307:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204014.428763:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204014.428768:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1788204014.429458:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204014.429460:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1788204014.429722:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204014.429724:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1788204014.429900:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204014.429901:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1788204014.430166:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204014.430168:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00010000:00000001:0.0:1788204015.260203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204015.260210:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204015.260214:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204015.260221:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204015.260230:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204015.260234:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204015.260240:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204015.260247:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204015.260252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204015.260254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204015.260258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204015.260260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204015.260264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204015.260265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204015.260324:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204015.260329:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204015.260332:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204015.772179:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320374533, at 4320374592 00000100:00000010:2.0:1788204015.772192:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc840. 00010000:00000001:0.0:1788204016.284195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204016.284200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204016.284209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204016.284211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204016.284215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204016.284219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204016.284225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204016.284227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204017.308213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204017.308218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204017.308226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204017.308230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204017.308237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204017.308240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204017.308247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204017.308249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204017.308301:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204017.308305:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204017.308308:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204018.332196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204018.332201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204018.332208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204018.332210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204018.332215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204018.332217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204018.332222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204018.332225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204018.332274:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204018.332277:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204018.332280:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204019.356186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204019.356189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204019.356195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204019.356198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204019.356200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204019.356203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204019.356207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204019.356210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204019.356273:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204019.356277:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204019.356280:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204019.548238:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204019.548249:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204019.548252:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204019.548255:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204019.548259:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204019.548267:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25705 00000400:00000200:0.0:1788204019.548276:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204019.548283:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204019.548286:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204019.548289:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548292:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.548295:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548297:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204019.548301:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204019.548304:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204019.548306:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204019.548310:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204019.548311:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204019.548313:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204019.548315:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548316:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.548317:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548318:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204019.548320:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204019.548322:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548323:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204019.548325:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204019.548327:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204019.548329:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548331:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204019.548333:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204019.548342:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204019.548346:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204019.548349:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204019.548351:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548354:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204019.548356:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204019.548360:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204019.548369:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204019.548376:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204019.548385:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543375744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204019.548394:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204019.548397:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204019.548400:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543375744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204019.548422:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204019.548424:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204019.548426:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204019.548427:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204019.548428:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204019.548430:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204019.548432:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25705 00000400:00000200:0.0:1788204019.548435:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204019.548437:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204019.548439:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204019.548441:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548443:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.548444:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548446:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204019.548447:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204019.548448:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204019.548450:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204019.548451:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204019.548452:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204019.548454:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204019.548458:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548460:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.548464:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548468:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204019.548469:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204019.548473:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548477:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204019.548482:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204019.548488:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204019.548492:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548494:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204019.548499:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204019.548503:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548506:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204019.548510:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204019.548515:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204019.548518:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204019.548519:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548522:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204019.548524:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204019.548527:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204019.548532:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788204019.548534:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204019.548541:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543375872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204019.548549:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204019.548551:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204019.548554:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543375872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204019.548562:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204019.548565:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204019.548569:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204019.548571:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204019.548573:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204019.548576:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204019.548580:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25705 00000400:00000200:0.0:1788204019.548587:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204019.548591:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204019.548594:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204019.548600:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548603:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.548606:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548607:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204019.548610:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204019.548613:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204019.548614:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204019.548615:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204019.548617:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204019.548620:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204019.548624:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548626:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.548629:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548633:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204019.548638:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204019.548646:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548653:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204019.548658:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204019.548663:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204019.548667:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548672:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204019.548677:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204019.548682:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548686:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204019.548692:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204019.548697:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000800:00000200:3.1:1788204019.548703:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:0.0:1788204019.548704:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000800:00000200:3.1:1788204019.548706:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 02000000:00000001:0.0:1788204019.548707:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548712:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204019.548714:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204019.548718:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204019.548724:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788204019.548731:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204019.548741:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543376000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204019.548752:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204019.548757:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204019.548764:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543376000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204019.548788:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204019.548791:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204019.548794:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204019.548798:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204019.548800:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204019.548805:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204019.548811:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25705 00000400:00000200:0.0:1788204019.548818:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204019.548824:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204019.548828:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204019.548835:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548840:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.548843:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548848:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204019.548851:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204019.548853:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204019.548857:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204019.548861:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204019.548866:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204019.548871:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204019.548878:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548881:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.548886:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548890:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204019.548895:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204019.548899:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204019.548904:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204019.548908:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204019.548913:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204019.548919:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548923:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204019.548927:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204019.548931:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548935:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204019.548941:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204019.548946:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204019.548952:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204019.548955:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.548959:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204019.548962:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204019.548966:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204019.548973:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788204019.548980:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204019.548991:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543376128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204019.549007:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204019.549011:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204019.549018:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543376128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204019.549032:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204019.549037:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204019.549044:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204019.549049:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204019.549052:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204019.549068:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204019.549074:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25705 00000400:00000200:0.0:1788204019.549082:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204019.549088:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204019.549132:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204019.549139:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.549144:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.549149:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.549153:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204019.549159:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204019.549163:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204019.549167:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204019.549171:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204019.549177:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204019.549182:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204019.549190:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.549195:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204019.549202:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204019.549207:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204019.549210:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204019.549217:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204019.549222:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204019.549225:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204019.549230:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204019.549234:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.549236:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204019.549241:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204019.549245:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.549248:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204019.549254:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204019.549259:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204019.549263:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204019.549267:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204019.549271:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204019.549274:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204019.549277:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204019.549284:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204019.549291:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204019.549300:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543376256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204019.549317:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204019.549322:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204019.549330:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543376256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204019.549349:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204019.549354:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25716) 00000800:00000200:3.1:1788204019.549386:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204019.549388:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204019.549637:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204019.549639:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204019.549901:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204019.549903:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204019.550184:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204019.550186:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204020.380224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204020.380229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204020.380238:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204020.380240:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204020.380243:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380246:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204020.380248:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380250:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380261:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204020.380270:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204020.380271:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204020.380273:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380275:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204020.380276:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380278:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380279:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204020.380285:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204020.380286:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204020.380288:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380289:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204020.380290:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380292:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204020.380293:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204020.380345:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204020.380349:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204020.380352:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204020.892158:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320379653, at 4320379712 00000100:00000010:2.0:1788204020.892172:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc360. 00010000:00000001:0.0:1788204021.404220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204021.404228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204021.404233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204021.404240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204021.404257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204021.404263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204021.404270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204021.404275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204021.404341:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204021.404345:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204021.404349:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204022.428186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204022.428191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204022.428199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204022.428202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204022.428218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204022.428220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204022.428224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204022.428225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204022.428266:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204022.428269:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204022.428272:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204023.452182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204023.452186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204023.452196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204023.452199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204023.452204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204023.452207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204023.452214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204023.452218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204023.452276:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204023.452280:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204023.452283:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204024.476168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204024.476173:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204024.476180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204024.476184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204024.476189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204024.476192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204024.476196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204024.476198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204024.476239:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204024.476244:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204024.476247:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204024.668188:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204024.668196:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204024.668198:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204024.668201:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204024.668204:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204024.668211:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25711 00000400:00000200:0.0:1788204024.668222:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204024.668229:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204024.668232:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204024.668235:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668238:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668241:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668244:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204024.668247:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204024.668249:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204024.668252:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204024.668255:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204024.668257:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204024.668260:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204024.668263:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668264:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668266:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668268:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204024.668271:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204024.668274:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668276:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204024.668278:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204024.668281:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204024.668283:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668285:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204024.668288:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204024.668295:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204024.668298:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204024.668306:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204024.668308:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668310:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204024.668313:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204024.668317:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204024.668322:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204024.668328:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204024.668335:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543376384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204024.668343:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204024.668347:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204024.668350:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543376384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204024.668366:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204024.668368:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204024.668371:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204024.668372:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204024.668373:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204024.668378:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204024.668380:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25711 00000400:00000200:0.0:1788204024.668383:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204024.668386:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204024.668388:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204024.668390:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668392:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668394:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668395:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204024.668398:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204024.668401:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204024.668403:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204024.668407:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204024.668410:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204024.668412:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204024.668419:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668423:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668427:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668430:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204024.668432:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204024.668436:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668440:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204024.668444:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204024.668450:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204024.668454:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668456:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204024.668460:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204024.668464:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668468:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204024.668472:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204024.668476:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204024.668480:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204024.668482:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668485:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204024.668487:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204024.668489:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204024.668493:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204024.668496:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204024.668502:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543376512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204024.668511:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204024.668515:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204024.668521:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543376512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204024.668545:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204024.668548:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204024.668552:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204024.668554:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204024.668556:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204024.668558:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204024.668561:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25711 00000400:00000200:0.0:1788204024.668565:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204024.668568:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204024.668570:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204024.668572:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668573:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668576:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668580:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204024.668582:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204024.668583:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204024.668584:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204024.668585:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204024.668588:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204024.668591:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204024.668595:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668598:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668601:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668603:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204024.668606:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204024.668612:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668615:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204024.668618:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204024.668621:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204024.668624:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668628:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204024.668631:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204024.668634:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668636:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204024.668638:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204024.668640:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204024.668644:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204024.668646:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668650:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204024.668651:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204024.668652:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204024.668657:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204024.668661:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204024.668666:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543376640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204024.668676:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204024.668689:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204024.668694:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543376640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204024.668699:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204024.668703:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204024.668707:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204024.668708:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204024.668711:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.1:1788204024.668712:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00080000:0.0:1788204024.668713:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204024.668718:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204024.668723:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25711 00000400:00000200:0.0:1788204024.668728:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204024.668732:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204024.668735:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204024.668740:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668742:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668746:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668749:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204024.668753:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204024.668755:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204024.668757:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204024.668758:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204024.668761:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204024.668764:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204024.668767:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668771:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668774:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668777:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204024.668781:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204024.668784:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668788:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204024.668790:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204024.668793:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204024.668797:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668800:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204024.668803:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204024.668807:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668810:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204024.668812:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204024.668814:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204024.668824:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204024.668826:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668828:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204024.668830:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204024.668832:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204024.668838:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204024.668841:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204024.668847:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543376768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204024.668852:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204024.668853:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204024.668855:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543376768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204024.668864:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204024.668866:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204024.668870:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204024.668872:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204024.668873:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204024.668878:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204024.668880:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25711 00000400:00000200:0.0:1788204024.668885:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204024.668887:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204024.668890:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204024.668893:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668897:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668904:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668907:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204024.668911:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204024.668912:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204024.668915:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204024.668918:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204024.668921:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204024.668923:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204024.668926:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668930:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204024.668932:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668935:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204024.668937:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204024.668941:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204024.668942:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204024.668943:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204024.668945:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204024.668946:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668947:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204024.668949:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204024.668952:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668956:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204024.668958:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204024.668961:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204024.668964:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204024.668965:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204024.668968:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204024.668970:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204024.668972:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204024.668975:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204024.668978:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204024.668984:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543376896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204024.669004:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204024.669007:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204024.669011:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543376896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204024.669021:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204024.669024:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25721) 00000800:00000200:3.1:1788204024.669281:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204024.669284:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204024.669565:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204024.669568:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204024.669856:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204024.669858:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204024.670134:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204024.670137:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204025.500249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204025.500258:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204025.500263:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204025.500273:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204025.500287:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204025.500295:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204025.500301:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204025.500307:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204025.500314:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204025.500317:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204025.500332:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204025.500335:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204025.500340:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204025.500342:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204025.500397:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204025.500401:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204025.500403:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204026.012227:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320384773, at 4320384832 00000100:00000010:2.0:1788204026.012248:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcc60. 00010000:00000001:0.0:1788204026.524235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204026.524241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204026.524250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204026.524254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204026.524261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204026.524265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204026.524271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204026.524275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204026.524343:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204026.524349:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204026.524352:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204027.548199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204027.548206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204027.548215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204027.548218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204027.548224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204027.548227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204027.548231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204027.548233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204027.548303:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204027.548308:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204027.548312:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204028.572211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204028.572217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204028.572226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204028.572230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204028.572235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204028.572237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204028.572240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204028.572242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204028.572308:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204028.572313:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204028.572316:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204029.596230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204029.596232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204029.596237:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204029.596238:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204029.596240:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204029.596244:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596245:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596247:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204029.596249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204029.596250:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204029.596251:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596252:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204029.596252:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204029.596258:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204029.596259:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204029.596259:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596260:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204029.596261:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596262:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204029.596263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204029.596335:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204029.596340:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204029.596344:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204029.788195:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204029.788203:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204029.788205:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204029.788207:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204029.788211:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204029.788216:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25716 00000400:00000200:0.0:1788204029.788224:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204029.788229:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204029.788231:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204029.788234:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788236:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788238:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788239:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204029.788242:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204029.788244:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204029.788246:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204029.788249:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204029.788250:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204029.788251:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204029.788253:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788254:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788255:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788256:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204029.788258:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204029.788260:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788262:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204029.788263:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204029.788266:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204029.788268:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788269:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204029.788271:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204029.788277:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204029.788279:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204029.788282:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204029.788283:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788285:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204029.788287:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204029.788290:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204029.788295:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204029.788300:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204029.788306:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543377024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204029.788317:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204029.788319:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204029.788322:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543377024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204029.788343:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204029.788345:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204029.788347:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204029.788348:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204029.788349:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204029.788351:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204029.788353:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25716 00000400:00000200:0.0:1788204029.788355:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204029.788357:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204029.788358:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204029.788360:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788361:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788363:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788364:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204029.788366:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204029.788366:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204029.788367:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204029.788369:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204029.788370:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204029.788371:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204029.788374:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788375:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788376:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788378:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204029.788380:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204029.788384:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788387:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204029.788389:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204029.788395:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204029.788397:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788401:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204029.788405:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204029.788409:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788413:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204029.788416:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204029.788419:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204029.788423:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204029.788425:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788429:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204029.788431:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204029.788433:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204029.788437:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204029.788445:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204029.788451:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543377152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204029.788459:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204029.788461:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204029.788464:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543377152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204029.788474:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204029.788479:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204029.788483:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204029.788486:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204029.788488:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204029.788489:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204029.788493:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25716 00000400:00000200:0.0:1788204029.788496:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204029.788498:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204029.788502:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204029.788507:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788510:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788513:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788515:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204029.788519:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204029.788521:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204029.788524:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204029.788526:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204029.788528:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204029.788532:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204029.788538:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788540:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788544:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788546:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204029.788550:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204029.788555:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788556:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204029.788557:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204029.788560:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204029.788563:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788564:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204029.788565:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204029.788568:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788572:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204029.788576:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204029.788580:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204029.788584:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204029.788588:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788601:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204029.788604:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204029.788607:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204029.788617:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204029.788621:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204029.788628:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543377280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204029.788639:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204029.788642:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204029.788648:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543377280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204029.788659:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204029.788664:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204029.788667:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204029.788668:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204029.788671:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.1:1788204029.788672:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00080000:0.0:1788204029.788673:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204029.788678:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204029.788681:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25716 00000400:00000200:0.0:1788204029.788687:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204029.788692:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204029.788694:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204029.788696:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788698:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788701:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788704:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204029.788707:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204029.788708:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204029.788709:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204029.788711:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204029.788712:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204029.788720:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204029.788723:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788725:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788727:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788731:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204029.788733:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204029.788737:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788739:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204029.788740:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204029.788742:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204029.788745:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788747:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204029.788749:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204029.788753:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788757:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204029.788760:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204029.788764:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204029.788767:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204029.788768:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788769:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204029.788770:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204029.788771:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204029.788773:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204029.788775:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204029.788782:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543377408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204029.788801:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204029.788805:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204029.788812:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543377408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204029.788824:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204029.788829:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204029.788834:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204029.788837:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204029.788840:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204029.788844:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204029.788848:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25716 00000400:00000200:0.0:1788204029.788853:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204029.788859:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204029.788864:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204029.788868:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788873:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788878:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788883:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204029.788887:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204029.788890:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204029.788893:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204029.788896:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204029.788899:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204029.788902:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204029.788908:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788912:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204029.788916:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788921:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204029.788925:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204029.788937:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204029.788942:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204029.788947:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204029.788950:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204029.788954:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788958:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204029.788962:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204029.788966:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788970:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204029.788974:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204029.788978:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204029.788984:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204029.788985:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204029.788994:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204029.788997:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204029.789000:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204029.789006:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204029.789010:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204029.789016:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543377536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204029.789031:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204029.789035:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204029.789043:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543377536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204029.789054:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204029.789060:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25726) 00000800:00000200:3.1:1788204029.789218:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204029.789220:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204029.789412:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204029.789414:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204029.789612:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204029.789614:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204029.789829:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204029.789831:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1788204030.620194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204030.620198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204030.620205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204030.620209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204030.620214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204030.620218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204030.620225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204030.620229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204030.620292:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204030.620296:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204030.620299:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204031.132154:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320389893, at 4320389952 00000100:00000010:2.0:1788204031.132164:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc120. 00010000:00000001:0.0:1788204031.644170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204031.644174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204031.644180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204031.644190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204031.644193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204031.644195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204031.644199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204031.644200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204031.644256:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204031.644261:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204031.644265:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204032.668186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204032.668192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204032.668200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204032.668205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204032.668211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204032.668216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204032.668223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204032.668226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204032.668284:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204032.668288:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204032.668291:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204033.692202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204033.692205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204033.692212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204033.692215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204033.692220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204033.692222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204033.692225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204033.692228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204033.692350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204033.692354:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204033.692357:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204034.716192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204034.716197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204034.716203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204034.716206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204034.716214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204034.716218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204034.716223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204034.716232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204034.716315:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204034.716319:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204034.716323:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204034.908804:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204034.908813:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204034.909377:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204034.909379:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204034.909655:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204034.909658:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204034.909914:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204034.909916:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204034.910193:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204034.910196:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00010000:00000001:0.0:1788204035.740168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204035.740174:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204035.740177:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204035.740182:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204035.740186:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204035.740190:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204035.740193:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204035.740197:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204035.740204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204035.740207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204035.740210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204035.740212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204035.740215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204035.740216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204035.740262:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204035.740264:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204035.740266:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204036.252190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320395013, at 4320395072 00000100:00000010:2.0:1788204036.252205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc540. 00010000:00000001:0.0:1788204036.764164:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204036.764173:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204036.764178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204036.764180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204036.764187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204036.764190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204036.764195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204036.764198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204036.764248:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204036.764252:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204036.764255:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204037.788210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204037.788215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204037.788221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204037.788225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204037.788230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204037.788233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204037.788237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204037.788240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204037.788302:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204037.788305:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204037.788308:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204038.812223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204038.812229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204038.812238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204038.812241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204038.812248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204038.812253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204038.812257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204038.812260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204038.812336:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204038.812341:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204038.812344:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204039.836182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204039.836188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204039.836197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204039.836201:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204039.836210:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836215:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204039.836217:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836223:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836227:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204039.836237:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204039.836239:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204039.836241:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836243:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204039.836244:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836245:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836246:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204039.836249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204039.836250:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204039.836251:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836252:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204039.836253:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204039.836256:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204039.836304:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204039.836307:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204039.836309:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204040.028580:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204040.028780:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204040.028914:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204040.029084:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204040.029252:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204040.860184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204040.860190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204040.860197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204040.860200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204040.860205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204040.860208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204040.860213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204040.860216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204040.860296:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204040.860300:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204040.860303:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204041.372165:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320400133, at 4320400192 00000100:00000010:2.0:1788204041.372178:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00010000:00000001:0.0:1788204041.884180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204041.884186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204041.884194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204041.884198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204041.884205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204041.884209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204041.884213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204041.884217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204041.884270:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204041.884274:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204041.884277:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204042.908192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204042.908196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204042.908204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204042.908207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204042.908209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204042.908211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204042.908215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204042.908218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204042.908281:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204042.908285:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204042.908288:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788204043.100340:0:256121:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:0.0:1788204043.100388:0:256121:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204043.100395:0:256121:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204043.100398:0:256121:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:0.0:1788204043.100403:0:256121:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204043.100407:0:256121:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:0.0:1788204043.100412:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204043.100416:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97a00. 00000400:00000200:0.0:1788204043.100422:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204043.100426:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204043.100432:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a65d8 00000400:00000010:0.0:1788204043.100435:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a65d8. 00000100:00000001:0.0:1788204043.100439:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204043.100448:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3ad80 x1875055543378304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100474:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204043.100479:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543378304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100485:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204043.100487:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204043.100497:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204043.100500:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:0.0:1788204043.100504:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204043.100508:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204043.100511:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6440 00000400:00000010:0.0:1788204043.100513:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6440. 00000100:00000001:0.0:1788204043.100515:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204043.100519:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a300 x1875055543378432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100529:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204043.100532:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543378432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100544:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204043.100548:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204043.100557:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204043.100561:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:0.0:1788204043.100567:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204043.100575:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204043.100585:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6dd0 00000400:00000010:0.0:1788204043.100589:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6dd0. 00000100:00000001:0.0:1788204043.100594:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204043.100602:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38a80 x1875055543378560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100616:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204043.100623:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543378560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100644:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204043.100648:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204043.100659:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204043.100662:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000400:00000200:0.0:1788204043.100668:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204043.100675:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204043.100679:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6e58 00000400:00000010:0.0:1788204043.100682:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6e58. 00000100:00000001:0.0:1788204043.100684:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204043.100690:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39180 x1875055543378688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100711:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204043.100717:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543378688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100728:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204043.100732:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204043.100741:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204043.100746:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000400:00000200:0.0:1788204043.100750:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204043.100756:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204043.100762:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a64c8 00000400:00000010:0.0:1788204043.100767:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a64c8. 00000100:00000001:0.0:1788204043.100770:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204043.100776:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38000 x1875055543378816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100791:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204043.100797:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543378816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204065 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204043.100813:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204043.100818:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1788204043.932171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204043.932176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204043.932182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204043.932184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204043.932188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204043.932191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204043.932195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204043.932198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204043.932238:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204043.932241:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204043.932244:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204044.956196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204044.956201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204044.956209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204044.956213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204044.956220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204044.956224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204044.956228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204044.956233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204044.956295:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204044.956300:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204044.956304:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204045.148775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204045.148781:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204045.149409:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204045.149416:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204045.149765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204045.149767:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204045.150129:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204045.150131:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204045.150404:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204045.150407:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204045.980234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204045.980249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204045.980261:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204045.980267:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204045.980273:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204045.980280:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204045.980285:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204045.980291:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204045.980301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204045.980305:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204045.980329:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204045.980333:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204045.980339:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204045.980340:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204045.980411:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204045.980416:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204045.980420:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204046.492199:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320405253, at 4320405312 00000100:00000010:2.0:1788204046.492218:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00010000:00000001:0.0:1788204047.004192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204047.004196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204047.004205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204047.004209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204047.004215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204047.004219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204047.004227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204047.004231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204047.004332:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204047.004336:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204047.004339:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204048.028193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204048.028197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204048.028203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204048.028204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204048.028207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204048.028209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204048.028212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204048.028213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204048.028269:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204048.028293:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204048.028297:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204049.052174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204049.052179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204049.052188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204049.052192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204049.052199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204049.052203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204049.052210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204049.052213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204049.052282:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204049.052287:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204049.052289:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204050.076170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204050.076175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204050.076186:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204050.076189:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204050.076193:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076196:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204050.076199:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076202:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076206:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204050.076214:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204050.076218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204050.076221:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076223:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204050.076226:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076228:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204050.076235:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204050.076236:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204050.076237:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076238:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204050.076248:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076249:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204050.076251:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204050.076300:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204050.076303:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204050.076305:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204050.268204:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204050.268214:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204050.268216:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204050.268220:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204050.268224:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204050.268231:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25736 00000400:00000200:0.0:1788204050.268266:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204050.268274:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204050.268277:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204050.268280:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268283:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.268285:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268288:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204050.268291:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204050.268294:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204050.268296:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204050.268299:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204050.268301:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204050.268304:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204050.268307:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268309:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.268311:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268313:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204050.268315:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204050.268318:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268321:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204050.268323:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204050.268327:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204050.268329:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268332:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204050.268334:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204050.268342:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204050.268345:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204050.268349:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204050.268350:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268353:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204050.268356:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204050.268360:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204050.268366:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204050.268372:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204050.268379:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543379584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204050.268386:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204050.268388:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204050.268391:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543379584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204050.268411:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204050.268413:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204050.268415:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204050.268416:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204050.268417:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204050.268419:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204050.268421:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25736 00000400:00000200:0.0:1788204050.268424:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204050.268426:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204050.268427:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204050.268429:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268431:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.268432:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268434:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204050.268435:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204050.268436:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204050.268438:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204050.268440:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204050.268442:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204050.268444:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204050.268448:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268453:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.268457:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268462:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204050.268466:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204050.268472:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268479:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204050.268484:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204050.268492:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204050.268503:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268509:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204050.268515:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204050.268520:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268524:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204050.268530:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204050.268535:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204050.268541:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204050.268544:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268546:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204050.268549:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204050.268553:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204050.268559:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204050.268566:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204050.268574:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543379712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204050.268585:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204050.268589:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204050.268594:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543379712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204050.268610:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204050.268616:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204050.268624:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204050.268627:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204050.268632:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204050.268636:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204050.268642:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25736 00000400:00000200:0.0:1788204050.268650:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204050.268656:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204050.268660:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204050.268665:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268667:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.268671:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268675:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204050.268677:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204050.268679:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204050.268682:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204050.268685:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204050.268688:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204050.268691:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204050.268697:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268701:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.268705:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268710:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204050.268715:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204050.268721:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268726:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204050.268728:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204050.268734:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204050.268738:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268742:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204050.268746:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204050.268751:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268752:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204050.268757:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204050.268760:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204050.268763:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204050.268766:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268771:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204050.268773:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204050.268777:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204050.268782:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000010:0.0:1788204050.268784:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000800:00000200:1.1:1788204050.268788:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1788204050.268790:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204050.268799:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543379840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204050.268810:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204050.268814:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204050.268822:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543379840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204050.268848:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204050.268854:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204050.268859:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204050.268863:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204050.268866:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204050.268872:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204050.268878:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25736 00000400:00000200:0.0:1788204050.268886:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204050.268891:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204050.268895:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204050.268902:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268906:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.268911:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268915:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204050.268919:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204050.268921:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204050.268923:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204050.268925:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204050.268929:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204050.268931:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204050.268937:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268940:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.268945:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268949:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204050.268952:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204050.268957:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204050.268962:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204050.268965:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204050.268969:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204050.268973:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268977:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204050.268980:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204050.268986:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.268988:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204050.268993:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204050.268997:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204050.269002:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204050.269005:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.269009:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204050.269010:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204050.269014:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204050.269018:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204050.269023:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204050.269030:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543379968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204050.269038:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204050.269042:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204050.269048:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543379968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204050.269059:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204050.269062:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204050.269067:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204050.269069:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204050.269071:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204050.269074:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204050.269078:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25736 00000400:00000200:0.0:1788204050.269084:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204050.269118:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204050.269121:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204050.269123:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.269126:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.269127:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.269129:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204050.269130:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204050.269131:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204050.269133:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204050.269135:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204050.269138:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204050.269140:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204050.269144:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.269148:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204050.269151:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204050.269155:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204050.269158:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204050.269163:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204050.269166:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204050.269170:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204050.269173:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204050.269175:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.269178:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204050.269181:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204050.269183:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.269187:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204050.269190:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204050.269194:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204050.269198:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204050.269200:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204050.269203:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204050.269205:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204050.269209:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204050.269212:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204050.269216:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204050.269221:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543380096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204050.269252:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204050.269255:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204050.269262:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543380096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204050.269272:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204050.269279:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25746) 00000800:00000200:1.1:1788204050.269454:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204050.269457:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204050.269742:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204050.269745:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204050.270005:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204050.270007:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204050.270308:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204050.270310:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788204051.100240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204051.100246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204051.100254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204051.100258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204051.100265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204051.100266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204051.100271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204051.100273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204051.100306:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204051.100310:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204051.100313:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204051.612232:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320410373, at 4320410432 00000100:00000010:2.0:1788204051.612252:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16474e0. 00010000:00000001:0.0:1788204052.124231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204052.124239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204052.124249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204052.124259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204052.124271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204052.124278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204052.124290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204052.124294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204052.124352:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204052.124357:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204052.124361:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204053.148165:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204053.148171:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204053.148179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204053.148183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204053.148202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204053.148203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204053.148207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204053.148208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204053.148259:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204053.148262:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204053.148264:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204054.172192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204054.172197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204054.172206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204054.172209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204054.172217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204054.172221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204054.172230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204054.172234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204054.172281:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204054.172284:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204054.172287:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204055.196199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204055.196204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204055.196215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204055.196219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204055.196227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204055.196231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204055.196245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204055.196248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204055.196309:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204055.196313:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204055.196317:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204055.388213:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204055.388225:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204055.388228:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204055.388231:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204055.388236:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204055.388243:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25741 00000400:00000200:2.0:1788204055.388253:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204055.388260:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204055.388263:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204055.388267:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388271:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.388273:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388275:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204055.388280:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204055.388281:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204055.388284:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204055.388287:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204055.388288:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204055.388290:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204055.388292:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388293:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.388294:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388296:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204055.388297:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204055.388300:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388301:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204055.388303:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204055.388306:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204055.388308:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388310:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204055.388312:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204055.388321:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204055.388324:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204055.388329:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204055.388330:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388333:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204055.388336:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204055.388339:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204055.388347:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204055.388353:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204055.388370:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543380224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204055.388384:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204055.388388:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204055.388393:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543380224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204055.388412:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204055.388416:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204055.388419:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204055.388420:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204055.388422:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204055.388425:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204055.388428:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25741 00000400:00000200:2.0:1788204055.388439:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204055.388443:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204055.388447:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204055.388453:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388459:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.388464:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388470:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204055.388476:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204055.388480:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204055.388484:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204055.388489:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204055.388501:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204055.388507:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204055.388515:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388518:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.388521:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388527:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204055.388533:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204055.388540:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388544:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204055.388549:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204055.388557:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204055.388563:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388567:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204055.388572:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204055.388577:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388582:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204055.388589:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204055.388595:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204055.388600:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204055.388603:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388608:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204055.388612:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204055.388614:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204055.388619:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204055.388625:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204055.388635:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543380352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204055.388654:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204055.388659:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204055.388667:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543380352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204055.388687:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204055.388691:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204055.388695:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204055.388698:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204055.388699:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204055.388704:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1788204055.388707:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204055.388709:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25741 00000800:00000200:1.1:1788204055.388711:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000400:00000200:2.0:1788204055.388716:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204055.388723:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204055.388725:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204055.388730:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388733:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.388734:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388735:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204055.388736:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204055.388737:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204055.388738:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204055.388739:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204055.388742:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204055.388744:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204055.388748:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388752:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.388754:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388756:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204055.388758:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204055.388762:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388767:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204055.388769:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204055.388771:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204055.388772:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388773:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204055.388775:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204055.388777:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388778:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204055.388782:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204055.388786:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204055.388791:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204055.388793:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388796:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204055.388797:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204055.388798:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204055.388800:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204055.388805:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204055.388810:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543380480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204055.388823:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204055.388828:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204055.388837:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543380480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204055.388854:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204055.388860:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204055.388865:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204055.388877:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204055.388880:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204055.388888:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204055.388895:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25741 00000400:00000200:2.0:1788204055.388905:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204055.388914:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204055.388917:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204055.388924:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388927:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.388931:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388934:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204055.388937:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204055.388939:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204055.388941:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204055.388943:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204055.388946:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204055.388948:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204055.388953:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388957:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.388960:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388965:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204055.388968:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204055.388972:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204055.388977:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204055.388981:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204055.388985:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204055.388989:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.388992:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204055.388995:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204055.388999:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.389002:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204055.389008:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204055.389011:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204055.389015:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204055.389018:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.389021:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204055.389023:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204055.389026:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204055.389032:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204055.389036:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204055.389043:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543380608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204055.389061:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204055.389066:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204055.389074:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543380608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204055.389126:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204055.389132:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204055.389139:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204055.389143:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204055.389162:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204055.389169:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204055.389175:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25741 00000400:00000200:2.0:1788204055.389182:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204055.389189:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204055.389196:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204055.389204:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.389209:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.389212:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.389215:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204055.389218:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204055.389221:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204055.389223:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204055.389227:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204055.389231:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204055.389233:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204055.389235:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.389237:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204055.389241:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204055.389246:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204055.389249:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204055.389251:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204055.389257:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204055.389260:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204055.389264:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204055.389269:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.389273:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204055.389277:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204055.389282:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.389286:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204055.389294:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204055.389298:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204055.389302:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204055.389306:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204055.389311:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204055.389316:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204055.389321:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204055.389329:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204055.389335:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204055.389343:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543380736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204055.389352:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204055.389356:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204055.389361:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543380736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204055.389379:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204055.389385:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25751) 00000800:00000200:1.1:1788204055.389408:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204055.389411:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204055.389598:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204055.389600:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204055.389786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204055.389787:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204055.390044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204055.390046:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1788204056.220199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204056.220205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204056.220210:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204056.220214:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204056.220219:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204056.220222:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204056.220226:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204056.220229:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204056.220234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204056.220238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204056.220244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204056.220247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204056.220252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204056.220253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204056.220312:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204056.220316:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204056.220320:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204056.732213:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320415493, at 4320415552 00000100:00000010:2.0:1788204056.732230:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00010000:00000001:0.0:1788204057.244196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204057.244202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204057.244213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204057.244216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204057.244221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204057.244224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204057.244229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204057.244233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204057.244293:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204057.244297:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204057.244299:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204058.268202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204058.268207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204058.268215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204058.268218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204058.268222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204058.268223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204058.268227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204058.268230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204058.268292:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204058.268296:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204058.268299:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204059.292180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204059.292185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204059.292195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204059.292198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204059.292206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204059.292210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204059.292218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204059.292221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204059.292284:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204059.292287:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204059.292289:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204060.316187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204060.316193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204060.316206:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204060.316209:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204060.316213:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204060.316218:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316220:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316225:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204060.316232:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204060.316235:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204060.316236:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316238:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204060.316239:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316242:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316244:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204060.316249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204060.316250:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204060.316251:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316253:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204060.316254:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316256:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204060.316258:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204060.316322:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204060.316326:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204060.316330:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204060.508199:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204060.508209:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204060.508212:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204060.508215:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204060.508219:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204060.508226:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25746 00000400:00000200:0.0:1788204060.508235:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204060.508241:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204060.508245:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204060.508248:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508250:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.508252:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508254:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204060.508258:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204060.508260:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204060.508263:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204060.508266:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204060.508268:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204060.508270:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204060.508272:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508274:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.508275:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508277:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204060.508280:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204060.508283:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508285:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204060.508288:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204060.508291:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204060.508293:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508295:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204060.508297:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204060.508306:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204060.508309:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204060.508312:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204060.508314:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508317:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204060.508319:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204060.508323:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204060.508330:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204060.508335:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204060.508343:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543380864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204060.508352:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204060.508355:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204060.508359:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543380864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204060.508384:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204060.508387:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204060.508390:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204060.508392:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204060.508393:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204060.508395:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204060.508398:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25746 00000400:00000200:0.0:1788204060.508402:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204060.508404:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204060.508406:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204060.508408:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508411:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.508412:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508415:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204060.508417:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204060.508418:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204060.508420:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204060.508422:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204060.508424:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204060.508427:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204060.508433:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508436:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.508440:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508443:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204060.508448:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204060.508452:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508457:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204060.508461:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204060.508468:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204060.508472:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508475:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204060.508481:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204060.508487:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508491:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204060.508496:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204060.508501:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204060.508506:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204060.508508:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508512:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204060.508515:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204060.508519:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204060.508525:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204060.508532:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204060.508546:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543380992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204060.508556:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204060.508560:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204060.508565:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543380992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204060.508580:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204060.508584:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204060.508590:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204060.508593:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204060.508596:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204060.508600:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204060.508604:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25746 00000400:00000200:0.0:1788204060.508610:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204060.508615:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204060.508620:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204060.508626:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508628:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.508633:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508637:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204060.508641:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204060.508644:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204060.508648:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204060.508652:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204060.508658:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204060.508664:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204060.508672:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508674:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.508676:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508678:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204060.508683:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204060.508689:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508694:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204060.508697:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204060.508701:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204060.508706:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508711:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204060.508713:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1788204060.508718:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000008:00000001:0.0:1788204060.508719:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508724:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204060.508724:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000010:0.0:1788204060.508732:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204060.508737:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204060.508742:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204060.508744:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508746:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204060.508749:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204060.508752:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204060.508758:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204060.508763:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204060.508771:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543381120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204060.508776:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204060.508777:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204060.508780:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543381120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204060.508785:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204060.508790:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204060.508795:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204060.508798:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204060.508801:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204060.508806:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204060.508811:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25746 00000400:00000200:0.0:1788204060.508821:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204060.508829:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204060.508833:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204060.508836:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508841:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.508843:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508847:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204060.508851:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204060.508852:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204060.508854:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204060.508856:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204060.508859:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204060.508861:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204060.508863:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508867:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.508868:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508869:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204060.508871:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204060.508873:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204060.508877:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204060.508878:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204060.508881:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204060.508882:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508886:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204060.508889:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204060.508894:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508895:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204060.508898:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204060.508901:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204060.508905:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204060.508907:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.508911:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204060.508912:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204060.508914:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204060.508918:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204060.508922:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204060.508929:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543381248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204060.508939:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204060.508943:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204060.508950:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543381248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204060.508964:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204060.508969:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204060.508978:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204060.508981:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204060.508982:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204060.508987:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204060.508991:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25746 00000400:00000200:0.0:1788204060.508996:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204060.508999:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204060.509000:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204060.509003:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.509007:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.509010:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.509013:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204060.509016:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204060.509018:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204060.509020:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204060.509023:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204060.509025:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204060.509027:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204060.509030:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.509034:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204060.509037:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204060.509041:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204060.509044:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204060.509048:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204060.509051:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204060.509054:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204060.509078:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204060.509083:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.509085:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204060.509088:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204060.509123:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.509125:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204060.509130:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204060.509133:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204060.509138:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204060.509140:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204060.509145:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204060.509148:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204060.509151:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204060.509157:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204060.509163:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204060.509179:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543381376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204060.509207:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204060.509212:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204060.509220:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543381376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204060.509232:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204060.509236:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25757) 00000800:00000200:1.1:1788204060.509351:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204060.509353:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204060.509626:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204060.509629:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204060.509901:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204060.509904:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204060.510194:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204060.510197:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00010000:00000001:0.0:1788204061.340192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204061.340198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204061.340208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204061.340212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204061.340220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204061.340224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204061.340233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204061.340238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204061.340299:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204061.340303:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204061.340307:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204061.852174:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320420613, at 4320420672 00000100:00000010:2.0:1788204061.852186:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00010000:00000001:0.0:1788204062.364191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204062.364203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204062.364209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204062.364211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204062.364215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204062.364216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204062.364222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204062.364224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204062.364283:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204062.364292:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204062.364295:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204063.388180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204063.388184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204063.388192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204063.388196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204063.388202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204063.388205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204063.388210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204063.388219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204063.388265:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204063.388273:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204063.388276:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204064.412186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204064.412192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204064.412199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204064.412203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204064.412210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204064.412215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204064.412226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204064.412229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204064.412282:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204064.412286:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204064.412289:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204065.436159:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204065.436163:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204065.436169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204065.436172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204065.436175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204065.436179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204065.436184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204065.436187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204065.436231:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204065.436233:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204065.436235:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204065.628201:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204065.628209:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204065.628211:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204065.628214:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204065.628216:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204065.628222:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25752 00000400:00000200:0.0:1788204065.628230:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204065.628235:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204065.628237:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204065.628239:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628241:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628243:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628244:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204065.628247:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204065.628248:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204065.628250:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204065.628253:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204065.628254:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204065.628256:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204065.628257:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628259:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628260:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628261:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204065.628263:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204065.628265:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628267:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204065.628269:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204065.628272:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204065.628274:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628275:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204065.628277:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204065.628284:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204065.628286:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204065.628289:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204065.628290:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628292:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204065.628294:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204065.628297:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204065.628303:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204065.628308:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204065.628314:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543381504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204065.628321:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204065.628324:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204065.628327:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543381504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204065.628346:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204065.628348:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204065.628350:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204065.628351:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204065.628352:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204065.628354:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204065.628355:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25752 00000400:00000200:0.0:1788204065.628358:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204065.628359:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204065.628361:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204065.628363:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628364:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628366:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628367:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204065.628369:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204065.628370:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204065.628371:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204065.628372:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204065.628374:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204065.628375:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204065.628376:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628377:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628379:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628381:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204065.628384:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204065.628387:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628390:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204065.628392:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204065.628398:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204065.628402:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628405:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204065.628410:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204065.628414:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628417:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204065.628421:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204065.628424:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204065.628428:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204065.628430:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628435:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204065.628439:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204065.628440:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204065.628443:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788204065.628449:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204065.628455:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543381632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204065.628463:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204065.628468:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204065.628476:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543381632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204065.628491:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204065.628496:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204065.628505:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204065.628509:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204065.628512:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204065.628516:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204065.628522:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25752 00000400:00000200:0.0:1788204065.628526:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204065.628529:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204065.628532:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204065.628536:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628538:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628541:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628543:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204065.628546:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204065.628547:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204065.628547:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204065.628549:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204065.628551:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204065.628555:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204065.628559:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628560:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628563:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628566:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204065.628568:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204065.628571:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628573:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204065.628574:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204065.628576:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204065.628579:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628582:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204065.628584:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204065.628586:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628589:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204065.628592:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204065.628595:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204065.628597:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204065.628600:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628603:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204065.628607:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204065.628610:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204065.628613:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788204065.628618:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204065.628624:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543381760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204065.628638:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204065.628641:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204065.628646:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543381760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204065.628656:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204065.628658:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204065.628664:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204065.628666:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204065.628668:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204065.628671:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1788204065.628672:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204065.628675:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25752 00000800:00000200:1.1:1788204065.628677:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000400:00000200:0.0:1788204065.628682:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204065.628689:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204065.628691:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204065.628695:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628696:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628700:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628705:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204065.628709:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204065.628709:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204065.628710:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204065.628711:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204065.628712:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204065.628713:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204065.628717:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628719:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628723:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628727:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204065.628730:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204065.628734:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628737:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204065.628738:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204065.628741:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204065.628745:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628748:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204065.628751:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204065.628754:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628758:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204065.628760:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204065.628762:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204065.628765:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204065.628767:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628768:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204065.628770:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204065.628772:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204065.628775:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788204065.628779:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204065.628785:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543381888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204065.628795:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204065.628796:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204065.628799:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543381888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204065.628807:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204065.628810:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204065.628815:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204065.628815:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204065.628817:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204065.628821:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204065.628824:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25752 00000400:00000200:0.0:1788204065.628828:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204065.628831:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204065.628833:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204065.628835:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628838:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628840:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628842:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204065.628843:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204065.628844:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204065.628845:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204065.628845:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204065.628846:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204065.628848:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204065.628853:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628858:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204065.628861:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628864:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204065.628869:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204065.628873:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204065.628876:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204065.628878:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204065.628882:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204065.628886:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628889:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204065.628892:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204065.628894:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628896:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204065.628901:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204065.628904:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204065.628907:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204065.628910:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204065.628913:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204065.628916:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204065.628918:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204065.628922:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204065.628926:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204065.628933:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543382016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204065.628954:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204065.628957:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204065.628963:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543382016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204065.628984:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204065.628987:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25762) 00000800:00000200:1.1:1788204065.629206:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204065.629208:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204065.629455:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204065.629456:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204065.629674:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204065.629676:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204065.629828:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204065.629830:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00010000:00000001:0.0:1788204066.460175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204066.460181:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204066.460185:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204066.460190:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204066.460196:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204066.460205:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204066.460208:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204066.460213:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204066.460220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204066.460222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204066.460226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204066.460228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204066.460232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204066.460234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204066.460282:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204066.460286:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204066.460289:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204066.972197:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320425733, at 4320425792 00000100:00000010:2.0:1788204066.972215:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. 00010000:00000001:0.0:1788204067.484189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204067.484195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204067.484203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204067.484207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204067.484215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204067.484219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204067.484225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204067.484228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204067.484296:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204067.484300:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204067.484303:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204068.508184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204068.508190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204068.508198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204068.508201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204068.508208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204068.508213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204068.508222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204068.508225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204068.508281:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204068.508285:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204068.508287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204069.532198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204069.532203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204069.532216:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204069.532219:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204069.532223:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204069.532234:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532237:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204069.532247:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204069.532248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204069.532249:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532251:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204069.532252:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204069.532259:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204069.532260:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204069.532261:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532262:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204069.532263:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532265:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204069.532266:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204069.532314:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204069.532318:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204069.532321:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204070.556178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204070.556183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204070.556197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204070.556201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204070.556211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204070.556214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204070.556218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204070.556221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204070.556287:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204070.556291:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204070.556294:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204070.748153:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204070.748163:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204070.748166:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204070.748170:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204070.748174:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204070.748181:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25757 00000400:00000200:0.0:1788204070.748190:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204070.748196:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204070.748200:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204070.748203:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748206:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.748208:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748210:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204070.748214:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204070.748216:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204070.748219:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204070.748222:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204070.748225:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204070.748227:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204070.748230:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748232:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.748234:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748236:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204070.748239:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204070.748242:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748244:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204070.748248:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204070.748251:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204070.748254:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748256:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204070.748259:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204070.748268:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204070.748272:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204070.748275:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204070.748277:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748279:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204070.748281:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204070.748284:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204070.748290:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204070.748295:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204070.748309:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543382144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204070.748316:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204070.748320:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204070.748323:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543382144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204070.748338:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204070.748341:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204070.748343:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204070.748344:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204070.748345:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204070.748347:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204070.748349:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25757 00000400:00000200:0.0:1788204070.748351:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204070.748354:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204070.748356:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204070.748358:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748360:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.748362:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748365:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204070.748368:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204070.748370:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204070.748373:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204070.748376:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204070.748380:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204070.748385:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204070.748391:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748396:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.748400:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748406:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204070.748410:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204070.748415:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748417:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204070.748420:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204070.748425:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204070.748429:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748431:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204070.748435:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204070.748441:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748445:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204070.748450:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204070.748452:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204070.748456:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204070.748459:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748462:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204070.748466:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204070.748469:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204070.748474:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204070.748480:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204070.748486:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543382272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204070.748495:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204070.748497:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204070.748500:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543382272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204070.748538:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204070.748543:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204070.748550:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204070.748552:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204070.748554:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204070.748558:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204070.748563:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25757 00000400:00000200:0.0:1788204070.748594:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204070.748600:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204070.748604:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204070.748608:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748611:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.748614:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748615:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204070.748624:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204070.748626:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204070.748629:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204070.748632:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204070.748633:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204070.748635:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204070.748640:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748643:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.748645:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748647:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204070.748650:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204070.748653:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748656:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204070.748659:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1788204070.748660:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:0.0:1788204070.748662:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204070.748666:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748667:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.1:1788204070.748667:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000008:00100000:0.0:1788204070.748670:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204070.748680:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748685:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204070.748689:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204070.748695:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204070.748700:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204070.748703:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748706:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204070.748708:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204070.748710:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204070.748716:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204070.748720:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204070.748727:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543382400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204070.748747:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204070.748751:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204070.748756:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543382400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204070.748765:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204070.748769:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204070.748776:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204070.748778:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204070.748781:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204070.748784:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204070.748787:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25757 00000400:00000200:0.0:1788204070.748793:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204070.748798:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204070.748799:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204070.748802:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748803:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.748805:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748808:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204070.748809:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204070.748810:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204070.748812:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204070.748815:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204070.748817:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204070.748818:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204070.748821:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748823:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.748825:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748827:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204070.748830:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204070.748834:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748836:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204070.748839:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204070.748841:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204070.748844:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748847:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204070.748850:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204070.748852:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748856:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204070.748861:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204070.748863:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204070.748866:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204070.748868:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.748870:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204070.748873:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204070.748875:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204070.748891:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204070.748898:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204070.748907:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543382528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204070.748927:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204070.748931:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204070.748936:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543382528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204070.748946:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204070.748950:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204070.748954:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204070.748957:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204070.748958:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204070.748961:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204070.748966:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25757 00000400:00000200:0.0:1788204070.748972:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204070.748979:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204070.748985:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204070.748992:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.748997:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.749002:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.749006:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204070.749010:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204070.749013:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204070.749017:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204070.749021:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204070.749026:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204070.749032:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204070.749040:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.749044:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204070.749048:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204070.749051:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204070.749055:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204070.749059:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204070.749061:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204070.749064:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204070.749068:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204070.749071:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.749075:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204070.749076:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204070.749078:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.749081:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204070.749084:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204070.749088:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204070.749107:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204070.749108:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204070.749109:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204070.749110:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204070.749112:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204070.749115:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204070.749117:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204070.749120:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543382656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204070.749125:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204070.749127:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204070.749131:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543382656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204070.749139:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204070.749143:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25767) 00000800:00000200:1.1:1788204070.749257:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204070.749258:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204070.749431:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204070.749432:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204070.749632:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204070.749634:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204070.749832:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204070.749834:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00010000:00000001:0.0:1788204071.580185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204071.580190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204071.580199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204071.580203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204071.580210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204071.580214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204071.580223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204071.580227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204071.580280:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204071.580284:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204071.580287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204072.092154:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320430853, at 4320430912 00000100:00000010:2.0:1788204072.092166:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00010000:00000001:0.0:1788204072.604194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204072.604199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204072.604209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204072.604212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204072.604219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204072.604222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204072.604228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204072.604231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204072.604290:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204072.604293:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204072.604296:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204073.628275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204073.628283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204073.628296:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204073.628301:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204073.628307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204073.628311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204073.628318:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204073.628325:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204073.628416:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204073.628422:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204073.628426:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204074.652244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204074.652252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204074.652262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204074.652272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204074.652278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204074.652281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204074.652285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204074.652287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204074.652357:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204074.652363:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204074.652367:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204075.676234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204075.676247:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204075.676252:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204075.676256:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204075.676261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204075.676265:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204075.676275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204075.676280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204075.676293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204075.676297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204075.676305:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204075.676311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204075.676319:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204075.676321:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204075.676380:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204075.676384:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204075.676386:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204075.868221:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204075.868232:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204075.868235:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204075.868238:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204075.868242:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204075.868250:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25762 00000400:00000200:0.0:1788204075.868260:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204075.868266:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204075.868269:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204075.868271:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868274:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.868275:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868277:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204075.868280:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204075.868282:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204075.868284:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204075.868288:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204075.868289:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204075.868290:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204075.868292:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868293:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.868295:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868296:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204075.868298:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204075.868300:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868302:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204075.868305:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204075.868308:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204075.868310:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868312:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204075.868315:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204075.868325:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204075.868328:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204075.868332:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204075.868333:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868336:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204075.868338:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204075.868343:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204075.868352:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204075.868360:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204075.868371:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543382784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204075.868386:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204075.868392:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204075.868401:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543382784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204075.868425:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204075.868428:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204075.868432:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204075.868434:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204075.868435:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204075.868438:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204075.868442:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25762 00000400:00000200:0.0:1788204075.868446:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204075.868449:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204075.868452:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204075.868456:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868458:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.868461:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868463:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204075.868464:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204075.868465:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204075.868467:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204075.868469:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204075.868471:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204075.868472:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204075.868474:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868476:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.868479:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868480:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204075.868481:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204075.868483:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868484:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204075.868486:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204075.868493:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204075.868495:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868496:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204075.868500:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204075.868502:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868503:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204075.868507:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204075.868510:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204075.868512:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204075.868515:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868518:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204075.868520:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204075.868523:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204075.868531:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204075.868539:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204075.868551:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543382912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204075.868568:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204075.868573:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204075.868580:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543382912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204075.868602:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204075.868606:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204075.868616:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204075.868620:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204075.868624:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204075.868627:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204075.868632:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25762 00000400:00000200:0.0:1788204075.868636:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204075.868641:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204075.868643:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204075.868650:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868653:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.868656:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868657:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204075.868658:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204075.868659:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204075.868660:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204075.868661:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204075.868664:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204075.868667:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204075.868672:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868680:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.868682:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868684:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204075.868689:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204075.868694:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868704:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204075.868706:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204075.868709:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204075.868711:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868712:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204075.868714:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204075.868719:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868720:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204075.868723:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204075.868725:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204075.868732:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204075.868734:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868741:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204075.868745:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204075.868748:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204075.868753:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204075.868757:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204075.868763:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543383040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204075.868768:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204075.868772:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204075.868775:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543383040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204075.868816:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204075.868819:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204075.868827:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204075.868830:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204075.868832:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204075.868841:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204075.868848:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25762 00000400:00000200:0.0:1788204075.868855:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204075.868860:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204075.868867:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204075.868876:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868879:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.868880:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868881:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204075.868883:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204075.868885:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204075.868890:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204075.868893:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204075.868895:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204075.868897:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204075.868900:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868906:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.868914:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868917:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204075.868919:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204075.868921:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204075.868927:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204075.868929:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204075.868932:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204075.868933:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868941:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204075.868944:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204075.868947:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868951:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204075.868954:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204075.868965:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204075.868970:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204075.868976:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.868979:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204075.868982:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204075.868985:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204075.868991:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204075.868997:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204075.869012:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543383168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204075.869020:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204075.869022:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204075.869025:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543383168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204075.869030:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204075.869035:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204075.869040:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204075.869041:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204075.869044:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204075.869050:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204075.869057:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25762 00000400:00000200:0.0:1788204075.869064:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204075.869070:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204075.869078:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204075.869085:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.869087:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.869109:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.869111:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204075.869112:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204075.869113:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204075.869114:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204075.869115:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204075.869116:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204075.869118:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204075.869120:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.869121:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204075.869122:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204075.869123:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204075.869124:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204075.869125:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204075.869127:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204075.869128:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204075.869130:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204075.869131:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.869133:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204075.869135:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204075.869136:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.869137:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204075.869139:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204075.869141:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204075.869143:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204075.869143:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204075.869144:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204075.869145:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204075.869146:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204075.869151:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204075.869153:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204075.869160:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543383296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204075.869169:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204075.869172:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204075.869182:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543383296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788204075.869200:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204075.869201:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204075.869215:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000040:0.0:1788204075.869216:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25772) 00000800:00000200:1.1:1788204075.869598:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204075.869600:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204075.869889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204075.869892:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204075.870121:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204075.870124:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204075.870292:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204075.870294:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1788204076.700209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204076.700217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204076.700223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204076.700225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204076.700229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204076.700230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204076.700232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204076.700233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204076.700310:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204076.700315:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204076.700319:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204077.212198:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320435974, at 4320436032 00000100:00000010:2.0:1788204077.212215:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647540. 00010000:00000001:0.0:1788204077.724259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204077.724267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204077.724278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204077.724283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204077.724289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204077.724293:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204077.724302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204077.724306:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204077.724374:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204077.724378:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204077.724380:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204078.748170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204078.748174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204078.748178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204078.748179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204078.748182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204078.748183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204078.748187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204078.748188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204078.748246:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204078.748252:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204078.748256:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204079.772251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204079.772261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204079.772282:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204079.772288:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204079.772296:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772306:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204079.772318:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772335:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772348:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772356:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204079.772358:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204079.772360:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204079.772362:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772365:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204079.772368:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772371:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772374:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772378:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204079.772380:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204079.772381:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204079.772382:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772384:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204079.772385:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772387:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204079.772389:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204079.772454:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204079.772458:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204079.772461:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204080.796249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204080.796260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204080.796268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204080.796272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204080.796278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204080.796281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204080.796290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204080.796295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204080.796367:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204080.796372:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204080.796378:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204080.988812:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204080.988818:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204080.989242:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204080.989243:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204080.989434:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204080.989436:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204080.989633:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204080.989635:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204080.989820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204080.989822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788204081.820194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204081.820198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204081.820205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204081.820208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204081.820214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204081.820217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204081.820223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204081.820227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204081.820318:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204081.820326:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204081.820331:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204082.332212:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320441093, at 4320441152 00000100:00000010:2.0:1788204082.332231:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00010000:00000001:0.0:1788204082.844197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204082.844203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204082.844211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204082.844217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204082.844240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204082.844247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204082.844262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204082.844264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204082.844311:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204082.844314:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204082.844316:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204083.868204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204083.868208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204083.868219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204083.868222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204083.868227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204083.868229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204083.868235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204083.868237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204083.868304:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204083.868309:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204083.868313:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204084.892186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204084.892191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204084.892196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204084.892200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204084.892211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204084.892217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204084.892221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204084.892224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204084.892280:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204084.892287:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204084.892290:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204085.916191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204085.916198:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204085.916211:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204085.916219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204085.916225:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204085.916230:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204085.916235:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204085.916240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204085.916244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204085.916247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204085.916251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204085.916253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204085.916257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204085.916259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204085.916309:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204085.916311:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204085.916313:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204086.108172:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204086.108180:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204086.108182:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204086.108185:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204086.108188:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204086.108193:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25772 00000400:00000200:0.0:1788204086.108200:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204086.108206:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204086.108208:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204086.108212:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108215:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.108217:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108219:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204086.108223:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204086.108225:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204086.108228:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204086.108232:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204086.108235:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204086.108237:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204086.108240:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108242:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.108244:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108245:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204086.108248:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204086.108251:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108254:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204086.108256:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204086.108259:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204086.108262:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108264:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204086.108266:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204086.108274:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204086.108277:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204086.108281:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204086.108282:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108285:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204086.108287:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204086.108291:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204086.108298:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204086.108304:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204086.108312:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543384064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204086.108322:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204086.108326:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204086.108331:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543384064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204086.108350:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204086.108353:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204086.108355:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204086.108356:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204086.108357:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204086.108359:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204086.108361:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25772 00000400:00000200:0.0:1788204086.108364:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204086.108367:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204086.108369:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204086.108371:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108372:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.108374:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108376:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204086.108377:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204086.108380:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204086.108382:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204086.108385:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204086.108388:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204086.108390:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204086.108394:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108398:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.108402:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108405:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204086.108409:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204086.108413:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108417:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204086.108419:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204086.108425:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204086.108429:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108433:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204086.108437:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204086.108440:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108443:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204086.108447:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204086.108450:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204086.108452:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204086.108454:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108458:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204086.108459:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204086.108460:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204086.108465:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204086.108470:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204086.108476:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543384192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204086.108484:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204086.108488:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204086.108497:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543384192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204086.108525:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204086.108532:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204086.108544:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204086.108547:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204086.108550:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204086.108555:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204086.108560:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25772 00000400:00000200:0.0:1788204086.108576:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204086.108583:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204086.108586:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204086.108592:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108594:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.108598:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108601:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204086.108603:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204086.108605:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204086.108607:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204086.108608:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204086.108609:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204086.108610:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204086.108612:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108625:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.108627:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108629:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204086.108631:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204086.108632:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108634:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204086.108635:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204086.108644:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204086.108649:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204086.108653:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204086.108654:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.1:1788204086.108656:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000008:00100000:0.0:1788204086.108659:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204086.108664:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108669:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204086.108675:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204086.108680:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204086.108685:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204086.108688:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108692:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204086.108696:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204086.108707:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204086.108716:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204086.108723:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204086.108732:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543384320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204086.108748:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204086.108752:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204086.108758:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543384320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204086.108772:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204086.108777:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204086.108783:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204086.108785:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204086.108788:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204086.108792:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204086.108795:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25772 00000400:00000200:0.0:1788204086.108802:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204086.108806:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204086.108808:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204086.108810:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108814:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.108818:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108820:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204086.108825:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204086.108827:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204086.108831:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204086.108834:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204086.108836:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204086.108839:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204086.108843:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108847:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.108850:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108854:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204086.108856:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204086.108861:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204086.108865:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204086.108869:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204086.108873:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204086.108878:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108883:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204086.108887:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204086.108895:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108899:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204086.108905:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204086.108910:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204086.108915:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204086.108919:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.108924:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204086.108929:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204086.108933:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204086.108940:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204086.108947:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204086.108967:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543384448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204086.108981:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204086.108986:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204086.108996:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543384448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204086.109006:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204086.109012:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204086.109018:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204086.109022:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204086.109027:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204086.109033:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204086.109041:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25772 00000400:00000200:0.0:1788204086.109046:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204086.109050:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204086.109055:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204086.109060:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.109065:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.109071:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.109077:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204086.109083:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204086.109087:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204086.109128:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204086.109133:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204086.109137:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204086.109141:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204086.109148:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.109153:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204086.109159:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204086.109164:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204086.109169:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204086.109176:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204086.109183:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204086.109189:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204086.109194:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204086.109200:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.109204:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204086.109207:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204086.109213:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.109218:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204086.109225:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000800:00000200:1.1:1788204086.109229:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:0.0:1788204086.109231:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204086.109235:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204086.109236:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204086.109238:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204086.109239:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204086.109241:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204086.109245:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204086.109249:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204086.109254:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543384576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204086.109261:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204086.109263:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204086.109268:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543384576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788204086.109269:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:0.0:1788204086.109287:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204086.109296:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25782) 00000800:00000200:1.1:1788204086.109454:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204086.109455:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204086.109632:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204086.109634:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204086.109851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204086.109854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204086.940183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204086.940189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204086.940198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204086.940201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204086.940209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204086.940212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204086.940220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204086.940224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204086.940277:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204086.940280:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204086.940283:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204087.452182:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320446213, at 4320446272 00000100:00000010:2.0:1788204087.452198:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00010000:00000001:0.0:1788204087.964162:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204087.964167:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204087.964175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204087.964179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204087.964184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204087.964188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204087.964193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204087.964197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204087.964250:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204087.964253:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204087.964256:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204088.988325:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204088.988329:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204088.988333:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204088.988335:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204088.988338:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204088.988339:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204088.988342:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204088.988343:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204088.988391:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204088.988394:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204088.988396:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204090.012175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204090.012179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204090.012189:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204090.012192:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204090.012196:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012201:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204090.012204:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012208:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012213:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204090.012219:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204090.012221:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204090.012222:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012224:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204090.012225:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012226:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012227:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204090.012231:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204090.012232:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204090.012233:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204090.012235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204090.012237:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204090.012292:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204090.012296:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204090.012299:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204091.036187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204091.036192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204091.036197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204091.036200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204091.036207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204091.036210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204091.036217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204091.036221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204091.036267:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204091.036270:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204091.036272:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204091.228634:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204091.228640:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204091.229198:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204091.229203:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204091.229461:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204091.229462:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204091.229677:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204091.229678:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204091.229836:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204091.229837:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204092.060160:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204092.060164:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204092.060171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204092.060175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204092.060181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204092.060184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204092.060188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204092.060190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204092.060237:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204092.060240:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204092.060242:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204092.572168:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320451333, at 4320451392 00000100:00000010:2.0:1788204092.572183:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc240. 00010000:00000001:0.0:1788204093.084184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204093.084191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204093.084200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204093.084203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204093.084209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204093.084213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204093.084218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204093.084221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204093.084271:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204093.084274:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204093.084277:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204094.108195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204094.108200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204094.108208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204094.108212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204094.108216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204094.108220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204094.108223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204094.108227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204094.108315:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204094.108319:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204094.108321:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204095.132195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204095.132201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204095.132209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204095.132213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204095.132221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204095.132224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204095.132231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204095.132235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204095.132301:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204095.132305:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204095.132307:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204096.156204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204096.156211:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204096.156216:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204096.156227:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204096.156234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204096.156237:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204096.156240:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204096.156243:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204096.156249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204096.156251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204096.156255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204096.156256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204096.156259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204096.156260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204096.156303:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204096.156306:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204096.156308:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204096.348757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204096.348770:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204096.349416:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204096.349419:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204096.349691:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204096.349694:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204096.349918:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204096.349920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204096.350114:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204096.350116:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1788204097.180190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204097.180196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204097.180205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204097.180209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204097.180215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204097.180221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204097.180230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204097.180235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204097.180294:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204097.180298:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204097.180301:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204097.692190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320456453, at 4320456512 00000100:00000010:2.0:1788204097.692206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc4e0. 00010000:00000001:0.0:1788204098.204201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204098.204209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204098.204217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204098.204221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204098.204224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204098.204228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204098.204231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204098.204235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204098.204307:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204098.204312:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204098.204316:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204099.228224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204099.228235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204099.228245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204099.228257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204099.228271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204099.228279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204099.228287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204099.228289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204099.228371:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204099.228378:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204099.228382:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204100.252211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204100.252215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204100.252227:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204100.252230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204100.252235:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252243:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204100.252247:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204100.252265:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204100.252267:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204100.252269:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252271:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204100.252273:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204100.252283:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204100.252285:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204100.252286:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252288:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204100.252290:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252291:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204100.252294:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204100.252350:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204100.252356:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204100.252360:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204101.276198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204101.276203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204101.276209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204101.276212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204101.276220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204101.276223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204101.276231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204101.276234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204101.276302:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204101.276306:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204101.276308:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204101.468207:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204101.468216:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204101.468218:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204101.468221:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204101.468224:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204101.468230:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25787 00000400:00000200:0.0:1788204101.468238:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204101.468244:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204101.468246:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204101.468249:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468251:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.468253:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468255:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204101.468258:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204101.468259:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204101.468261:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204101.468264:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204101.468265:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204101.468267:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204101.468269:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468270:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.468272:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468273:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204101.468275:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204101.468277:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468279:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204101.468281:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204101.468284:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204101.468287:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468288:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204101.468291:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204101.468298:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204101.468301:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204101.468304:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204101.468306:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468308:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204101.468310:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204101.468313:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204101.468321:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788204101.468326:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204101.468333:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543385984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204101.468341:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204101.468345:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204101.468373:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543385984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204101.468392:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204101.468394:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204101.468397:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204101.468398:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204101.468399:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204101.468401:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204101.468403:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25787 00000400:00000200:0.0:1788204101.468406:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204101.468408:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204101.468410:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204101.468412:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468414:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.468416:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468418:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204101.468420:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204101.468422:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204101.468423:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204101.468425:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204101.468427:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204101.468428:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204101.468430:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468433:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.468436:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468438:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204101.468442:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204101.468447:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468451:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204101.468453:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204101.468459:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204101.468463:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468467:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204101.468472:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204101.468477:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468482:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204101.468486:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204101.468490:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204101.468496:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204101.468499:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468501:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204101.468505:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204101.468507:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204101.468511:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788204101.468514:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204101.468537:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543386112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204101.468550:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204101.468554:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204101.468559:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543386112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204101.468573:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204101.468579:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204101.468585:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204101.468587:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204101.468590:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204101.468595:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204101.468600:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25787 00000400:00000200:0.0:1788204101.468606:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204101.468612:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204101.468615:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204101.468618:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468622:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.468624:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468628:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204101.468631:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204101.468632:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204101.468633:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204101.468635:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204101.468639:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204101.468641:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204101.468646:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468648:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.468649:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468650:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204101.468655:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204101.468661:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468663:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204101.468665:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204101.468670:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204101.468673:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468675:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204101.468679:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204101.468682:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468685:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204101.468689:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204101.468693:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204101.468696:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204101.468699:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468704:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204101.468707:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204101.468710:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204101.468714:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788204101.468717:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204101.468726:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543386240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204101.468739:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204101.468742:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204101.468747:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543386240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204101.468761:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204101.468766:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204101.468772:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204101.468773:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204101.468776:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204101.468778:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.1:1788204101.468778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1788204101.468784:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204101.468791:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25787 00000400:00000200:0.0:1788204101.468797:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204101.468802:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204101.468806:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204101.468812:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468814:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.468818:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468822:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204101.468825:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204101.468827:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204101.468830:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204101.468831:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204101.468834:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204101.468836:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204101.468842:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468844:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.468847:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468851:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204101.468854:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204101.468859:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204101.468863:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204101.468864:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204101.468866:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204101.468870:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468871:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204101.468874:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204101.468876:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468879:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204101.468883:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204101.468887:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204101.468891:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204101.468893:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.468896:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204101.468898:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204101.468900:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204101.468904:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204101.468911:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204101.468922:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543386368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204101.468938:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204101.468941:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204101.468949:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543386368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204101.468958:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204101.468962:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204101.468969:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204101.468973:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204101.468978:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204101.468985:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204101.468992:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25787 00000400:00000200:0.0:1788204101.469001:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204101.469008:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204101.469013:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204101.469019:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.469025:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.469029:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.469033:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204101.469039:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204101.469042:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204101.469046:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204101.469051:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204101.469055:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204101.469060:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204101.469068:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.469075:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204101.469080:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204101.469083:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204101.469088:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204101.469117:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204101.469119:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204101.469121:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204101.469123:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204101.469126:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.469130:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204101.469133:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204101.469138:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.469142:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204101.469147:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204101.469150:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204101.469153:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204101.469155:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204101.469158:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204101.469161:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204101.469163:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204101.469171:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204101.469176:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204101.469184:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543386496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204101.469201:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204101.469206:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204101.469214:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543386496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204101.469224:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204101.469227:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25797) 00000800:00000200:3.1:1788204101.469498:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204101.469501:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204101.469829:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204101.469832:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204101.470163:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204101.470165:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204101.470475:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204101.470478:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000008:00000020:0.0:1788204101.980204:0:417027:0:(osc_request.c:745:osc_update_next_shrink()) next time 26993 to shrink grant 00000008:00000020:0.0:1788204101.980211:0:417027:0:(osc_request.c:745:osc_update_next_shrink()) next time 26993 to shrink grant 00010000:00000001:0.0:1788204102.300181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204102.300187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204102.300195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204102.300200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204102.300216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204102.300224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204102.300231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204102.300233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204102.300278:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204102.300283:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204102.300287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204102.812181:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320461573, at 4320461632 00000100:00000010:2.0:1788204102.812189:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc7e0. 00010000:00000001:0.0:1788204103.324205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204103.324209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204103.324216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204103.324219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204103.324223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204103.324225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204103.324231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204103.324234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204103.324287:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204103.324291:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204103.324293:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204104.348211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204104.348218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204104.348230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204104.348234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204104.348243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204104.348249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204104.348258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204104.348264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204104.348350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204104.348353:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204104.348356:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204105.372198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204105.372204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204105.372212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204105.372215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204105.372221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204105.372226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204105.372231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204105.372236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204105.372307:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204105.372312:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204105.372316:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204106.396188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204106.396195:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204106.396199:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204106.396204:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204106.396208:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204106.396213:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204106.396217:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204106.396222:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204106.396229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204106.396232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204106.396236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204106.396241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204106.396244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204106.396248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204106.396313:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204106.396318:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204106.396321:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204106.588224:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204106.588234:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204106.588236:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204106.588240:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204106.588245:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204106.588252:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25792 00000400:00000200:0.0:1788204106.588290:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204106.588297:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204106.588301:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204106.588305:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588308:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.588311:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588314:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204106.588318:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204106.588320:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204106.588323:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204106.588326:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204106.588329:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204106.588332:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204106.588335:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588337:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.588340:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588342:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204106.588345:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204106.588349:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588352:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204106.588355:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204106.588359:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204106.588362:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.588365:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204106.588367:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204106.588378:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204106.588391:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204106.588396:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204106.588398:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.588401:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204106.588404:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204106.588408:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204106.588416:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204106.588423:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204106.588433:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543386624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204106.588444:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204106.588449:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204106.588454:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543386624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204106.588481:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204106.588484:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204106.588488:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204106.588490:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204106.588492:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204106.588496:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204106.588499:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25792 00000400:00000200:0.0:1788204106.588503:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204106.588508:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204106.588511:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204106.588523:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588531:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.588536:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588542:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204106.588546:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204106.588547:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204106.588549:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204106.588552:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204106.588555:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204106.588560:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204106.588567:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588582:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.588588:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588592:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204106.588598:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204106.588606:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588611:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204106.588616:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204106.588623:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204106.588627:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.588629:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204106.588634:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204106.588639:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.588645:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204106.588653:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204106.588656:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204106.588662:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204106.588665:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.588667:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204106.588670:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204106.588674:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204106.588682:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204106.588688:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204106.588709:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543386752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204106.588730:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204106.588737:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204106.588748:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543386752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204106.588768:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204106.588773:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204106.588779:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204106.588782:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204106.588785:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204106.588790:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204106.588796:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25792 00000400:00000200:0.0:1788204106.588806:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204106.588814:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204106.588818:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204106.588821:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588824:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.588828:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588834:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204106.588841:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204106.588845:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204106.588849:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204106.588853:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204106.588860:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204106.588865:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204106.588873:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588879:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.588896:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588903:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204106.588906:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204106.588914:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204106.588920:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204106.588923:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204106.588929:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204106.588934:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204106.588939:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204106.588940:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204106.588944:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204106.588949:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.588951:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204106.588955:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204106.588958:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204106.588963:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204106.588967:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.588971:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204106.588975:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204106.588979:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204106.588987:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204106.588992:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204106.589007:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543386880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204106.589021:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204106.589024:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204106.589031:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543386880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204106.589048:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204106.589053:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204106.589059:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204106.589061:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204106.589063:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204106.589070:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204106.589078:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25792 00000400:00000200:0.0:1788204106.589089:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204106.589122:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204106.589126:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204106.589130:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589133:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.589139:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589143:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204106.589146:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204106.589148:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204106.589152:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204106.589156:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204106.589163:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204106.589169:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204106.589175:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589180:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.589182:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589189:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204106.589195:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204106.589201:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589206:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204106.589210:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204106.589214:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204106.589219:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.589223:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204106.589227:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204106.589232:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.589234:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204106.589240:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204106.589243:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204106.589247:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204106.589249:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.589254:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204106.589270:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204106.589273:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204106.589281:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204106.589290:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204106.589300:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543387008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204106.589318:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204106.589322:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204106.589333:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543387008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204106.589353:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204106.589358:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204106.589364:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204106.589369:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204106.589373:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204106.589378:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204106.589385:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25792 00000400:00000200:0.0:1788204106.589395:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204106.589407:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204106.589413:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204106.589423:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589429:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.589436:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589443:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204106.589449:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204106.589452:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204106.589457:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204106.589461:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204106.589468:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204106.589475:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204106.589484:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589489:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204106.589496:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788204106.589500:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204106.589501:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204106.589508:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204106.589515:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204106.589522:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204106.589527:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204106.589533:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204106.589539:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.589544:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204106.589548:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204106.589554:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.589559:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204106.589567:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204106.589573:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204106.589579:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204106.589582:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204106.589588:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204106.589591:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204106.589594:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204106.589601:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204106.589608:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204106.589617:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543387136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204106.589635:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204106.589640:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204106.589648:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543387136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204106.589660:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204106.589667:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25803) 00000800:00000200:3.1:1788204106.589769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204106.590028:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204106.590213:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204107.420172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204107.420177:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204107.420182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204107.420185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204107.420193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204107.420197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204107.420207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204107.420211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204107.420278:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204107.420283:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204107.420287:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204107.932204:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320466693, at 4320466752 00000100:00000010:2.0:1788204107.932220:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcea0. 00010000:00000001:0.0:1788204108.444209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204108.444216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204108.444247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204108.444252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204108.444260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204108.444264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204108.444268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204108.444271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204108.444323:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204108.444326:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204108.444328:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204109.468175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204109.468179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204109.468186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204109.468189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204109.468198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204109.468202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204109.468224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204109.468230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204109.468283:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204109.468287:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204109.468289:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204109.660406:0:256120:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788204109.660458:0:256120:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788204109.660470:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204109.660474:0:256120:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788204109.660479:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204109.660484:0:256120:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788204109.660497:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204109.660501:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:2.0:1788204109.660507:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204109.660512:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204109.660518:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987346e8 00000400:00000010:2.0:1788204109.660521:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987346e8. 00000100:00000001:2.0:1788204109.660527:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204109.660535:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75dad80 x1875055543386624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660562:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204109.660567:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dad80 x1875055543386624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660575:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204109.660578:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204109.660591:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204109.660594:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdd00. 00000400:00000200:2.0:1788204109.660597:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204109.660602:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204109.660606:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734ee0 00000400:00000010:2.0:1788204109.660608:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734ee0. 00000100:00000001:2.0:1788204109.660614:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204109.660623:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8000 x1875055543386752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660659:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204109.660667:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8000 x1875055543386752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660688:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204109.660694:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204109.660704:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204109.660711:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdb00. 00000400:00000200:2.0:1788204109.660716:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204109.660727:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204109.660738:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734000 00000400:00000010:2.0:1788204109.660743:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734000. 00000100:00000001:2.0:1788204109.660749:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204109.660759:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8700 x1875055543386880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660780:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204109.660789:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8700 x1875055543386880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660805:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204109.660810:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204109.660819:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204109.660825:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:2.0:1788204109.660829:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204109.660832:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204109.660835:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734440 00000400:00000010:2.0:1788204109.660837:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734440. 00000100:00000001:2.0:1788204109.660842:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204109.660852:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8380 x1875055543387008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660877:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204109.660886:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8380 x1875055543387008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660903:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204109.660909:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204109.660922:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204109.660928:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:2.0:1788204109.660934:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204109.660948:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204109.660953:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987342a8 00000400:00000010:2.0:1788204109.660958:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987342a8. 00000100:00000001:2.0:1788204109.660965:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204109.660974:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75db100 x1875055543387136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.660991:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204109.660999:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db100 x1875055543387136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204131 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204109.661013:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204109.661018:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1788204110.492209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204110.492213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204110.492226:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204110.492230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204110.492234:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204110.492245:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492248:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204110.492269:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204110.492272:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204110.492276:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492278:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204110.492280:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492282:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492284:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204110.492291:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204110.492293:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204110.492296:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492298:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204110.492300:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492302:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204110.492304:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204110.492365:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204110.492369:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204110.492372:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204111.516204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204111.516210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204111.516219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204111.516223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204111.516227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204111.516232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204111.516239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204111.516243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204111.516312:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204111.516316:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204111.516318:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204111.709500:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204111.709508:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204111.709896:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204111.709898:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204111.710245:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204111.710248:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204111.710556:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204111.710559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204111.712463:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204111.712467:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204112.540194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204112.540199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204112.540205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204112.540208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204112.540216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204112.540219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204112.540223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204112.540227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204112.540286:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204112.540292:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204112.540296:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204113.052176:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320471814, at 4320471872 00000100:00000010:2.0:1788204113.052189:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00010000:00000001:0.0:1788204113.564200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204113.564205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204113.564212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204113.564216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204113.564223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204113.564227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204113.564236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204113.564241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204113.564288:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204113.564291:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204113.564293:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204114.588184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204114.588189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204114.588195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204114.588198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204114.588202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204114.588212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204114.588218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204114.588221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204114.588273:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204114.588277:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204114.588280:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204115.612183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204115.612196:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204115.612199:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204115.612205:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204115.612211:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204115.612216:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204115.612217:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204115.612223:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204115.612232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204115.612235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204115.612238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204115.612239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204115.612242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204115.612244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204115.612298:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204115.612303:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204115.612307:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204116.636259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204116.636270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204116.636286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204116.636294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204116.636311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204116.636314:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204116.636321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204116.636323:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204116.636403:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204116.636408:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204116.636412:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204116.828873:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204116.828881:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204116.829532:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204116.829534:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204116.829724:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204116.829726:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204116.829996:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204116.829999:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204116.830228:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204116.830230:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788204117.660195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204117.660201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204117.660210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204117.660214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204117.660222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204117.660226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204117.660233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204117.660237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204117.660298:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204117.660302:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204117.660306:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204118.172177:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320476933, at 4320476992 00000100:00000010:2.0:1788204118.172192:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00010000:00000001:0.0:1788204118.684195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204118.684199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204118.684207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204118.684217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204118.684222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204118.684224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204118.684228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204118.684233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204118.684294:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204118.684299:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204118.684303:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204119.708183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204119.708188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204119.708196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204119.708201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204119.708205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204119.708209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204119.708219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204119.708223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204119.708294:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204119.708298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204119.708302:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204120.732204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204120.732209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204120.732225:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204120.732230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204120.732236:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204120.732246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732258:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204120.732266:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204120.732267:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204120.732268:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732270:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204120.732271:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732272:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732274:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204120.732279:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204120.732281:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204120.732282:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732284:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204120.732285:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732286:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204120.732288:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204120.732350:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204120.732353:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204120.732356:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204121.756169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204121.756175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204121.756182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204121.756187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204121.756195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204121.756198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204121.756202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204121.756206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204121.756290:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204121.756294:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204121.756296:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204121.948226:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204121.948238:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204121.948241:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204121.948245:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204121.948249:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204121.948256:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25808 00000400:00000200:0.0:1788204121.948267:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204121.948275:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204121.948278:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204121.948282:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948284:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.948287:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948288:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204121.948292:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204121.948294:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204121.948296:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204121.948299:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204121.948300:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204121.948302:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204121.948305:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948306:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.948307:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948309:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204121.948311:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204121.948313:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948315:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204121.948317:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204121.948320:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204121.948322:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948323:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204121.948326:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204121.948334:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204121.948337:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204121.948340:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204121.948341:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948343:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204121.948345:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204121.948349:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204121.948356:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204121.948361:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204121.948368:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543388544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204121.948376:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204121.948380:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204121.948383:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543388544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204121.948404:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204121.948406:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204121.948409:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204121.948410:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204121.948412:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204121.948414:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204121.948416:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25808 00000400:00000200:0.0:1788204121.948419:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204121.948422:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204121.948424:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204121.948427:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948429:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.948431:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948432:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204121.948435:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204121.948437:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204121.948440:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204121.948442:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204121.948446:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204121.948450:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204121.948455:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948457:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.948461:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948465:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204121.948470:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204121.948475:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948479:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204121.948483:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204121.948489:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204121.948492:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948498:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204121.948502:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204121.948507:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948509:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204121.948514:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204121.948518:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204121.948520:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204121.948523:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948524:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204121.948533:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204121.948536:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204121.948546:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204121.948552:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204121.948563:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543388672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204121.948577:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204121.948581:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204121.948589:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543388672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204121.948607:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204121.948611:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204121.948619:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204121.948622:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204121.948624:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204121.948629:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204121.948634:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25808 00000400:00000200:0.0:1788204121.948642:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204121.948647:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204121.948651:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204121.948656:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948659:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.948663:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948666:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204121.948668:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204121.948669:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204121.948672:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204121.948675:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204121.948680:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204121.948681:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204121.948684:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948686:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.948688:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948691:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204121.948695:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204121.948698:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948700:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204121.948701:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204121.948704:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204121.948707:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948712:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204121.948714:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204121.948717:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948720:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204121.948724:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204121.948728:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204121.948731:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204121.948734:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948737:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204121.948740:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204121.948742:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204121.948746:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204121.948752:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204121.948759:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543388800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204121.948773:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204121.948775:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204121.948779:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543388800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204121.948798:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204121.948801:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204121.948805:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204121.948809:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204121.948810:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204121.948812:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204121.948816:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25808 00000400:00000200:0.0:1788204121.948820:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204121.948824:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204121.948825:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204121.948827:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948828:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.948830:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948831:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204121.948833:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204121.948834:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204121.948836:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204121.948839:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204121.948844:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204121.948847:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204121.948851:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948856:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.948859:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948864:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.1:1788204121.948864:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:0.0:1788204121.948870:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000800:00000200:1.1:1788204121.948871:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00080000:0.0:1788204121.948876:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204121.948882:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204121.948886:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204121.948890:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204121.948893:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948898:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204121.948900:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204121.948904:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948907:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204121.948910:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204121.948915:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204121.948920:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204121.948922:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.948926:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204121.948927:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204121.948931:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204121.948936:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204121.948941:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204121.948949:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543388928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204121.948957:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204121.948959:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204121.948964:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543388928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204121.948989:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204121.949016:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204121.949019:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204121.949020:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204121.949021:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204121.949024:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204121.949028:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25808 00000400:00000200:0.0:1788204121.949037:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204121.949040:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204121.949044:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204121.949047:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.949052:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.949054:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.949055:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204121.949057:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204121.949059:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204121.949061:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204121.949065:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204121.949066:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204121.949068:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204121.949070:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.949075:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204121.949076:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204121.949078:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204121.949080:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204121.949087:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204121.949114:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204121.949116:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204121.949119:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204121.949121:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.949122:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204121.949124:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204121.949126:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.949127:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204121.949133:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204121.949136:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204121.949141:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204121.949143:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204121.949145:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204121.949148:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204121.949151:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204121.949155:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204121.949161:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204121.949168:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543389056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204121.949187:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204121.949191:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204121.949198:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543389056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204121.949208:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204121.949214:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25818) 00000800:00000200:1.1:1788204121.949528:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204121.949530:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204121.949854:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204121.949857:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204121.950226:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204121.950229:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204121.950537:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204121.950540:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1788204122.780195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204122.780199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204122.780205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204122.780208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204122.780215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204122.780217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204122.780220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204122.780223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204122.780290:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204122.780295:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204122.780299:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204123.292206:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320482053, at 4320482112 00000100:00000010:2.0:1788204123.292219:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647420. 00010000:00000001:0.0:1788204123.804203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204123.804209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204123.804217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204123.804222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204123.804229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204123.804233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204123.804242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204123.804246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204123.804275:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204123.804279:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204123.804282:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204124.828187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204124.828193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204124.828200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204124.828214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204124.828224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204124.828229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204124.828238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204124.828241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204124.828296:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204124.828299:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204124.828302:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204125.852196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204125.852202:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204125.852208:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204125.852212:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204125.852217:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204125.852220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204125.852226:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204125.852229:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204125.852236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204125.852241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204125.852243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204125.852248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204125.852251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204125.852254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204125.852308:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204125.852311:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204125.852314:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204126.876192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204126.876197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204126.876203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204126.876208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204126.876212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204126.876216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204126.876219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204126.876221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204126.876312:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204126.876316:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204126.876318:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204127.068805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204127.068813:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204127.069395:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204127.069397:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204127.069588:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204127.069590:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204127.069807:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204127.069809:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204127.070035:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204127.070037:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1788204127.900200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204127.900207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204127.900217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204127.900222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204127.900230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204127.900235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204127.900242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204127.900245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204127.900277:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204127.900281:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204127.900285:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204128.412196:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320487173, at 4320487232 00000100:00000010:2.0:1788204128.412212:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fccc0. 00010000:00000001:0.0:1788204128.924170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204128.924174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204128.924181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204128.924186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204128.924190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204128.924193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204128.924198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204128.924202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204128.924296:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204128.924301:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204128.924303:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204129.948199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204129.948205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204129.948216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204129.948220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204129.948227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204129.948231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204129.948237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204129.948241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204129.948294:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204129.948299:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204129.948301:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204130.972174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204130.972180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204130.972197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204130.972201:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204130.972207:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972215:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204130.972218:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972221:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972225:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204130.972230:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204130.972230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204130.972232:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204130.972235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972238:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204130.972242:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204130.972242:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204130.972243:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972244:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204130.972245:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972246:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204130.972247:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204130.972301:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204130.972307:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204130.972310:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204131.996186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204131.996195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204131.996206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204131.996211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204131.996219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204131.996224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204131.996231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204131.996236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204131.996298:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204131.996303:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204131.996307:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204132.188888:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204132.188896:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204132.189623:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204132.189625:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204132.189961:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204132.189965:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204132.190241:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204132.190243:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204132.190495:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204132.190497:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00010000:00000001:0.0:1788204133.020221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204133.020231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204133.020248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204133.020256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204133.020267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204133.020272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204133.020285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204133.020292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204133.020415:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204133.020422:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204133.020426:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204133.532231:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320492294, at 4320492352 00000100:00000010:2.0:1788204133.532250:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcd80. 00010000:00000001:0.0:1788204134.044204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204134.044210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204134.044217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204134.044226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204134.044240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204134.044247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204134.044261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204134.044267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204134.044344:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204134.044349:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204134.044353:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204135.068224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204135.068229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204135.068236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204135.068240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204135.068245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204135.068248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204135.068253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204135.068256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204135.068339:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204135.068345:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204135.068349:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204136.092234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204136.092245:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204136.092251:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204136.092258:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204136.092266:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204136.092272:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204136.092278:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204136.092285:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204136.092295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204136.092297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204136.092302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204136.092304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204136.092310:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204136.092312:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204136.092404:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204136.092409:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204136.092413:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204137.116227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204137.116234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204137.116245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204137.116249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204137.116258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204137.116263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204137.116270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204137.116276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204137.116382:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204137.116388:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204137.116391:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204137.308676:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204137.308679:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788204137.310370:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204137.310375:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788204137.311398:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204137.311409:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788204137.311723:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204137.311766:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788204137.312002:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204137.312005:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00010000:00000001:0.0:1788204138.140245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204138.140250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204138.140256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204138.140259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204138.140262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204138.140263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204138.140266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204138.140269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204138.140367:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204138.140374:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204138.140378:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204138.652217:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320497413, at 4320497472 00000100:00000010:2.0:1788204138.652232:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647960. 00010000:00000001:0.0:1788204139.164188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204139.164195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204139.164201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204139.164203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204139.164212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204139.164216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204139.164230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204139.164239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204139.164310:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204139.164317:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204139.164322:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204140.188190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204140.188195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204140.188201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204140.188203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204140.188207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204140.188210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204140.188215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204140.188218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204140.188288:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204140.188292:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204140.188296:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204141.212171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204141.212176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204141.212187:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204141.212191:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204141.212195:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212203:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204141.212206:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212209:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212214:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204141.212218:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204141.212219:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204141.212220:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212221:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204141.212223:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212226:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204141.212230:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204141.212231:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204141.212232:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204141.212235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204141.212237:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204141.212282:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204141.212285:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204141.212287:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204142.236181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204142.236189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204142.236198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204142.236202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204142.236210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204142.236216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204142.236226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204142.236230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204142.236288:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204142.236292:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204142.236294:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204142.428761:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204142.428815:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788204142.429550:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204142.429552:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788204142.429792:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204142.429795:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788204142.430032:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204142.430034:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788204142.430344:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204142.430346:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00010000:00000001:0.0:1788204143.260191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204143.260196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204143.260205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204143.260209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204143.260214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204143.260218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204143.260223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204143.260226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204143.772162:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320502533, at 4320502592 00000100:00000010:2.0:1788204143.772175:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. 00010000:00000001:0.0:1788204144.284198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204144.284203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204144.284211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204144.284215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204144.284219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204144.284222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204144.284226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204144.284231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204144.284286:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204144.284289:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204144.284292:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204145.308209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204145.308216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204145.308227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204145.308232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204145.308242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204145.308245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204145.308250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204145.308252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204145.308313:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204145.308316:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204145.308318:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204146.332193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204146.332200:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204146.332203:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204146.332210:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204146.332217:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204146.332220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204146.332223:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204146.332227:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204146.332233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204146.332235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204146.332237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204146.332238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204146.332240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204146.332241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204147.356198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204147.356202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204147.356208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204147.356212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204147.356216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204147.356219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204147.356224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204147.356227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204147.356301:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204147.356305:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204147.356309:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204147.548874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204147.548881:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204147.549584:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204147.549587:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204147.549916:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204147.549919:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204147.550266:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204147.550269:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204147.550751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204147.550754:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788204148.380187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204148.380193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204148.380200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204148.380204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204148.380207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204148.380211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204148.380215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204148.380218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204148.380291:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204148.380296:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204148.380300:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204148.892194:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320507653, at 4320507712 00000100:00000010:2.0:1788204148.892209:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fccc0. 00010000:00000001:0.0:1788204149.404205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204149.404210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204149.404215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204149.404219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204149.404223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204149.404225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204149.404228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204149.404231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204149.404307:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204149.404311:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204149.404316:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204150.428344:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204150.428353:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204150.428360:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204150.428362:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204150.428366:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204150.428367:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204150.428371:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204150.428372:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204150.428602:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204150.428606:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204150.428608:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204151.452196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204151.452202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204151.452218:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204151.452221:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204151.452226:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452232:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204151.452235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452240:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452242:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204151.452248:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204151.452249:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204151.452251:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452258:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204151.452260:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452262:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452264:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204151.452269:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204151.452270:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204151.452271:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452273:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204151.452274:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204151.452277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204151.452313:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204151.452317:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204151.452320:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204152.476194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204152.476199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204152.476204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204152.476207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204152.476215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204152.476218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204152.476224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204152.476228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204152.476294:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204152.476299:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204152.476302:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204152.668174:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204152.668181:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204152.668183:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204152.668185:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204152.668189:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204152.668194:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25839 00000400:00000200:2.0:1788204152.668201:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204152.668206:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204152.668208:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204152.668210:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668212:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668214:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668215:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204152.668218:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204152.668219:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204152.668221:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204152.668224:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204152.668225:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204152.668227:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204152.668229:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668230:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668231:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668232:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204152.668234:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204152.668237:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668238:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204152.668240:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204152.668243:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204152.668244:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668245:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204152.668247:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204152.668254:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204152.668256:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204152.668259:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204152.668260:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668261:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204152.668263:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204152.668267:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204152.668272:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204152.668277:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204152.668283:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543392384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204152.668290:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204152.668293:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204152.668296:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543392384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204152.668311:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204152.668313:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204152.668315:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204152.668316:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204152.668317:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204152.668319:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204152.668321:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25839 00000400:00000200:2.0:1788204152.668323:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204152.668325:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204152.668326:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204152.668328:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668329:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668331:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668332:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204152.668333:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204152.668334:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204152.668334:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204152.668336:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204152.668337:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204152.668338:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204152.668340:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668342:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668343:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668344:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204152.668346:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204152.668348:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668349:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204152.668351:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204152.668355:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204152.668357:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668358:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204152.668363:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204152.668367:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668369:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204152.668373:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204152.668375:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204152.668378:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204152.668379:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668381:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204152.668383:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204152.668385:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204152.668390:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204152.668394:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204152.668400:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543392512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204152.668409:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204152.668412:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204152.668417:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543392512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204152.668429:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204152.668433:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204152.668435:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204152.668435:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204152.668436:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204152.668438:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204152.668440:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25839 00000400:00000200:2.0:1788204152.668445:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204152.668449:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204152.668450:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204152.668454:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668457:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668487:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668491:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204152.668493:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204152.668495:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204152.668496:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204152.668498:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204152.668502:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204152.668505:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204152.668507:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668509:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668512:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668514:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204152.668516:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204152.668517:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668519:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204152.668520:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204152.668522:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204152.668524:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668525:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204152.668527:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204152.668529:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668530:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204152.668536:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204152.668539:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204152.668544:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204152.668546:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668549:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204152.668550:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204152.668552:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204152.668557:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204152.668565:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204152.668572:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543392640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204152.668583:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204152.668586:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204152.668588:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543392640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204152.668601:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204152.668606:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204152.668611:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204152.668614:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204152.668617:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204152.668631:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204152.668638:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25839 00000400:00000200:2.0:1788204152.668698:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204152.668704:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204152.668708:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204152.668713:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668716:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668720:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668725:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204152.668729:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204152.668732:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204152.668735:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:3.1:1788204152.668736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:2.0:1788204152.668738:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204152.668742:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:3.1:1788204152.668742:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1788204152.668746:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204152.668752:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668756:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668760:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668764:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204152.668767:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204152.668779:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668785:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204152.668789:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204152.668792:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204152.668797:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668801:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204152.668805:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204152.668810:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668813:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204152.668817:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204152.668821:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204152.668827:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204152.668830:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668834:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204152.668836:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204152.668838:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204152.668844:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204152.668849:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204152.668857:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543392768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204152.668863:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204152.668866:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204152.668872:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543392768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204152.668881:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204152.668883:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204152.668888:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204152.668890:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204152.668891:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204152.668893:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204152.668897:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25839 00000400:00000200:2.0:1788204152.668902:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204152.668906:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204152.668909:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204152.668913:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668916:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668920:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668922:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204152.668926:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204152.668928:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204152.668930:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204152.668934:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204152.668935:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204152.668938:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204152.668943:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668944:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204152.668948:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668951:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204152.668955:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204152.668960:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204152.668963:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204152.668967:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204152.668969:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204152.668973:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668976:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204152.668978:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204152.668982:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.668984:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204152.668989:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204152.668994:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204152.668996:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204152.668999:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204152.669000:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204152.669003:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204152.669004:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204152.669009:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204152.669018:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204152.669022:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543392896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204152.669034:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204152.669037:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204152.669043:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543392896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204152.669058:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204152.669065:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25849) 00000800:00000200:3.1:1788204152.669346:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204152.669349:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204152.669659:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204152.669661:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204152.669931:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204152.669934:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204152.670207:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204152.670210:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204153.500179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204153.500184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204153.500190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204153.500193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204153.500197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204153.500199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204153.500204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204153.500207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204153.500269:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204153.500273:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204153.500277:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204154.012184:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320512773, at 4320512832 00000100:00000010:2.0:1788204154.012200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc660. 00010000:00000001:0.0:1788204154.524239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204154.524246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204154.524263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204154.524269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204154.524276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204154.524284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204154.524298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204154.524306:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204154.524361:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204154.524364:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204154.524366:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204155.548310:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204155.548315:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204155.548318:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204155.548321:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204155.548324:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204155.548326:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204155.548328:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204155.548330:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204155.548334:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204155.548335:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204155.548338:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204155.548339:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204155.548342:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204155.548343:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204155.548392:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204155.548397:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204155.548399:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204156.572203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204156.572210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204156.572220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204156.572223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204156.572231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204156.572234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204156.572237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204156.572239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204156.572295:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204156.572298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204156.572300:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204157.596188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204157.596194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204157.596201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204157.596202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204157.596207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204157.596211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204157.596220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204157.596224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204157.596318:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204157.596323:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204157.596326:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204157.788252:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204157.788263:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204157.788265:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204157.788268:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204157.788272:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204157.788279:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25844 00000400:00000200:2.0:1788204157.788289:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204157.788305:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204157.788308:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204157.788311:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788313:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.788315:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788316:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204157.788320:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204157.788321:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204157.788324:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204157.788327:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204157.788329:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204157.788331:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204157.788332:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788334:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.788336:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788337:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204157.788339:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204157.788341:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788343:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204157.788344:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204157.788347:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204157.788348:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.788350:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204157.788352:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204157.788358:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204157.788361:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204157.788364:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204157.788365:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.788367:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204157.788379:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204157.788383:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204157.788389:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204157.788394:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204157.788400:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543393024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204157.788408:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204157.788411:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204157.788414:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543393024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204157.788427:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204157.788429:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204157.788431:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204157.788432:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204157.788433:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204157.788435:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204157.788444:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25844 00000400:00000200:2.0:1788204157.788447:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204157.788450:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204157.788451:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204157.788454:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788455:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.788456:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788458:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204157.788459:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204157.788460:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204157.788461:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204157.788463:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204157.788464:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204157.788468:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204157.788470:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788482:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.788487:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788493:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204157.788498:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204157.788503:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788508:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204157.788513:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204157.788527:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204157.788532:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.788535:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204157.788540:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204157.788546:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.788549:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204157.788554:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204157.788560:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204157.788566:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204157.788568:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.788571:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204157.788574:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204157.788578:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204157.788582:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204157.788587:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204157.788598:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543393152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204157.788618:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204157.788623:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204157.788640:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543393152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204157.788657:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204157.788669:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204157.788675:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204157.788678:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204157.788679:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204157.788681:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204157.788684:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25844 00000400:00000200:2.0:1788204157.788717:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204157.788725:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204157.788729:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204157.788733:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788738:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.788744:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788748:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204157.788753:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:3.1:1788204157.788753:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:2.0:1788204157.788755:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204157.788758:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204157.788760:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204157.788762:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204157.788764:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000800:00000200:3.1:1788204157.788770:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1788204157.788771:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788777:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.788782:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788792:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204157.788796:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204157.788801:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788804:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204157.788805:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204157.788807:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204157.788809:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.788810:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204157.788812:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204157.788816:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.788819:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204157.788822:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204157.788826:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204157.788830:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204157.788833:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.788838:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204157.788840:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204157.788844:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204157.788849:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204157.788857:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204157.788872:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543393280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204157.788886:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204157.788888:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204157.788892:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543393280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204157.788898:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204157.788904:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204157.788907:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204157.788911:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204157.788914:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204157.788919:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204157.788925:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25844 00000400:00000200:2.0:1788204157.788934:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204157.788942:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204157.788947:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204157.788951:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788954:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.788957:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788959:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204157.788962:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204157.788965:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204157.788968:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204157.788971:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204157.788975:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204157.788980:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204157.788986:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788989:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.788992:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.788996:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204157.789000:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204157.789005:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204157.789009:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204157.789014:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204157.789019:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204157.789025:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.789030:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204157.789034:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204157.789047:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.789051:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204157.789059:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204157.789063:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204157.789067:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204157.789069:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.789073:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204157.789075:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204157.789078:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204157.789084:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204157.789118:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204157.789129:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543393408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204157.789139:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204157.789142:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204157.789147:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543393408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204157.789156:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204157.789158:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204157.789162:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204157.789166:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204157.789170:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204157.789176:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204157.789180:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25844 00000400:00000200:2.0:1788204157.789187:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204157.789193:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204157.789198:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204157.789204:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.789209:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.789215:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.789219:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204157.789223:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204157.789227:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204157.789230:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204157.789234:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204157.789240:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204157.789244:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204157.789252:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.789256:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204157.789262:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204157.789267:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204157.789270:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204157.789275:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204157.789278:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204157.789280:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204157.789284:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204157.789288:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.789290:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204157.789292:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204157.789296:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.789298:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204157.789304:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204157.789308:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204157.789311:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204157.789312:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204157.789314:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204157.789315:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204157.789317:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204157.789323:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204157.789327:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204157.789333:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543393536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204157.789341:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204157.789344:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204157.789350:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543393536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204157.789355:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204157.789358:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25854) 00000800:00000200:3.1:1788204157.789421:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204157.789422:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204157.789635:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204157.789637:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204157.789838:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204157.789840:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204157.790118:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204157.790120:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204158.620182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204158.620187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204158.620195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204158.620198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204158.620203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204158.620206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204158.620209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204158.620212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204158.620275:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204158.620281:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204158.620285:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204159.132201:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320517893, at 4320517952 00000100:00000010:2.0:1788204159.132210:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf00. 00010000:00000001:0.0:1788204159.644205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204159.644210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204159.644219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204159.644223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204159.644229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204159.644233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204159.644239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204159.644243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204159.644291:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204159.644294:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204159.644296:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204160.669431:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204160.669439:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669455:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204160.669457:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204160.669459:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204160.669463:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669465:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204160.669467:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669469:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669471:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669474:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204160.669476:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204160.669477:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204160.669478:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669479:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204160.669481:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669482:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669484:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669487:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204160.669489:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204160.669490:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204160.669492:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669493:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204160.669494:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669496:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204160.669498:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204160.669577:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204160.669588:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204160.669591:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204161.692230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204161.692243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204161.692253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204161.692261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204161.692265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204161.692273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204161.692285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204161.692295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204161.692412:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204161.692417:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204161.692420:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204162.716193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204162.716199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204162.716204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204162.716208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204162.716216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204162.716220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204162.716231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204162.716234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204162.716337:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204162.716341:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204162.716343:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204162.908966:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204162.909008:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204162.909992:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204162.909995:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204162.910183:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204162.910185:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204162.910383:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204162.910385:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204162.910724:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204162.910725:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1788204163.740230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204163.740240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204163.740255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204163.740259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204163.740263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204163.740274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204163.740280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204163.740282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204163.740342:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204163.740345:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204163.740348:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204164.252183:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320523014, at 4320523072 00000100:00000010:2.0:1788204164.252201:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00010000:00000001:0.0:1788204164.764190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204164.764195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204164.764202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204164.764205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204164.764214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204164.764217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204164.764222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204164.764226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204164.764293:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204164.764297:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204164.764300:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204165.788220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204165.788227:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204165.788242:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204165.788259:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204165.788266:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204165.788270:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204165.788273:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204165.788276:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204165.788282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204165.788284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204165.788288:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204165.788290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204165.788293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204165.788294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204165.788350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204165.788353:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204165.788355:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204166.812186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204166.812191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204166.812195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204166.812198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204166.812204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204166.812206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204166.812223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204166.812225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204166.812277:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204166.812281:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204166.812283:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204167.836189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204167.836211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204167.836284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204167.836290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204167.836299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204167.836302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204167.836307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204167.836311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204167.836371:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204167.836375:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204167.836377:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204168.028865:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204168.028870:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204168.031288:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204168.031290:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204168.031490:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204168.032434:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204168.032619:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204168.032620:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204168.033791:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204168.033794:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00010000:00000001:0.0:1788204168.862492:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204168.862497:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204168.862501:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204168.862502:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204168.862504:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204168.862505:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204168.862507:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204168.862508:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204168.862596:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204168.862616:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204168.862620:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204169.372238:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320528134, at 4320528192 00000100:00000010:2.0:1788204169.372258:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00010000:00000001:0.0:1788204169.884175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204169.884180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204169.884190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204169.884201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204169.884212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204169.884215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204169.884227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204169.884231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204169.884299:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204169.884304:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204169.884307:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204170.908292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204170.908307:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908315:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204170.908321:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204170.908324:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204170.908328:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908334:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204170.908339:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908342:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908350:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908356:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204170.908359:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204170.908362:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204170.908367:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908370:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204170.908378:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908383:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908399:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908408:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204170.908411:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204170.908413:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204170.908415:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908417:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204170.908418:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908420:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204170.908422:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204170.908489:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204170.908493:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204170.908497:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204171.932257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204171.932265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204171.932271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204171.932275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204171.932281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204171.932283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204171.932289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204171.932293:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204171.932373:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204171.932379:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204171.932383:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204172.956224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204172.956230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204172.956238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204172.956241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204172.956248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204172.956251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204172.956258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204172.956261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204172.956399:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204172.956403:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204172.956407:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204173.149008:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204173.151012:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204173.151188:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204173.151325:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204173.151413:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204173.980207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204173.980212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204173.980218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204173.980223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204173.980229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204173.980232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204173.980238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204173.980241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204173.980296:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204173.980300:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204173.980303:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204174.492156:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320533253, at 4320533312 00000100:00000010:2.0:1788204174.492168:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647900. 00010000:00000001:0.0:1788204175.004222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204175.004229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204175.004240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204175.004244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204175.004250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204175.004253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204175.004259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204175.004263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204175.004344:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204175.004350:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204175.004353:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204176.028191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204176.028199:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204176.028202:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204176.028207:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204176.028215:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204176.028220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204176.028223:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204176.028230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204176.028237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204176.028241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204176.028249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204176.028252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204176.028256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204176.028257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204176.028313:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204176.028316:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204176.028318:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204176.220393:0:256122:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788204176.220435:0:256122:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788204176.220444:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204176.220448:0:256122:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788204176.220454:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204176.220458:0:256122:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788204176.220464:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204176.220468:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:2.0:1788204176.220473:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204176.220478:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204176.220484:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6a18 00000400:00000010:2.0:1788204176.220488:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6a18. 00000100:00000001:2.0:1788204176.220493:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204176.220505:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a680 x1875055543394944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220549:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204176.220554:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543394944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220562:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204176.220565:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204176.220577:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204176.220580:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:2.0:1788204176.220583:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204176.220586:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204176.220589:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6cc0 00000400:00000010:2.0:1788204176.220592:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6cc0. 00000100:00000001:2.0:1788204176.220597:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204176.220604:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39f80 x1875055543395072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220615:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204176.220623:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543395072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220634:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204176.220636:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204176.220647:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204176.220650:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000400:00000200:2.0:1788204176.220655:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204176.220661:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204176.220668:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6220 00000400:00000010:2.0:1788204176.220672:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6220. 00000100:00000001:2.0:1788204176.220677:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204176.220686:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3b480 x1875055543395200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220705:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204176.220713:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543395200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220725:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204176.220728:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204176.220739:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204176.220744:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000400:00000200:2.0:1788204176.220749:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204176.220757:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204176.220761:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a66e8 00000400:00000010:2.0:1788204176.220765:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a66e8. 00000100:00000001:2.0:1788204176.220771:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204176.220778:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39c00 x1875055543395328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220791:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204176.220798:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543395328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220809:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204176.220812:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204176.220833:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204176.220838:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:2.0:1788204176.220843:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204176.220851:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204176.220857:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:2.0:1788204176.220861:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:2.0:1788204176.220866:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204176.220874:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38000 x1875055543395456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220893:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204176.220899:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543395456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204198 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204176.220912:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204176.220915:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1788204177.052191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204177.052195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204177.052200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204177.052203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204177.052211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204177.052215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204177.052222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204177.052224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204177.052276:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204177.052280:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204177.052283:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204178.076177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204178.076182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204178.076187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204178.076190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204178.076197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204178.076200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204178.076203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204178.076205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204178.076254:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204178.076257:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204178.076259:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204178.268815:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204178.268822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204178.269558:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204178.269561:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204178.269856:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204178.269858:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204178.270050:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204178.270052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204178.270412:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204178.270415:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204179.100177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204179.100182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204179.100186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204179.100189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204179.100195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204179.100199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204179.100202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204179.100205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204179.100268:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204179.100271:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204179.100274:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204179.612189:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320538373, at 4320538432 00000100:00000010:2.0:1788204179.612202:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00010000:00000001:0.0:1788204180.124186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204180.124193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204180.124203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204180.124209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204180.124217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204180.124221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204180.124231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204180.124236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204180.124305:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204180.124309:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204180.124313:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204181.148171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204181.148176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204181.148186:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204181.148189:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204181.148195:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148201:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204181.148204:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148208:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148210:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204181.148215:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204181.148216:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204181.148217:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148218:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204181.148219:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148220:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148221:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204181.148225:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204181.148226:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204181.148226:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148227:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204181.148228:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148229:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204181.148230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204181.148285:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204181.148289:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204181.148293:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204182.172202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204182.172208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204182.172218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204182.172222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204182.172232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204182.172237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204182.172244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204182.172249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204182.172312:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204182.172315:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204182.172318:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204183.196208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204183.196214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204183.196224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204183.196228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204183.196235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204183.196239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204183.196248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204183.196253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204183.196323:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204183.196328:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204183.196331:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204183.388898:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204183.388904:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204183.389731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204183.389733:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204183.389954:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204183.389957:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204183.390208:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204183.390210:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204183.390551:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204183.390554:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788204184.220199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204184.220204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204184.220215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204184.220219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204184.220225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204184.220229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204184.220235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204184.220240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204184.220317:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204184.220320:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204184.220323:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204184.732179:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320543493, at 4320543552 00000100:00000010:2.0:1788204184.732191:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647720. 00010000:00000001:0.0:1788204185.244187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204185.244191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204185.244199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204185.244202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204185.244205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204185.244208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204185.244218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204185.244220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204185.244278:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204185.244280:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204185.244282:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204186.268193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204186.268200:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204186.268205:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204186.268210:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204186.268216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204186.268220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204186.268226:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204186.268231:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204186.268238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204186.268242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204186.268247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204186.268251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204186.268260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204186.268262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204186.268312:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204186.268315:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204186.268317:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204187.292169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204187.292174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204187.292182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204187.292186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204187.292193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204187.292197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204187.292203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204187.292206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204187.292258:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204187.292261:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204187.292263:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204188.316205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204188.316210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204188.316216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204188.316220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204188.316225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204188.316228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204188.316234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204188.316238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204188.316299:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204188.316304:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204188.316307:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204188.508298:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204188.508311:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204188.508313:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204188.508318:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204188.508324:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204188.508332:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25874 00000400:00000200:0.0:1788204188.508346:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204188.508355:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204188.508359:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204188.508362:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508365:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508367:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508369:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204188.508373:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204188.508375:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204188.508377:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204188.508381:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204188.508383:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204188.508384:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204188.508386:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508388:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508389:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508391:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204188.508395:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204188.508397:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508399:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204188.508403:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204188.508408:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204188.508411:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508413:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204188.508418:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204188.508432:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204188.508436:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204188.508440:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204188.508441:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508445:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204188.508448:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204188.508453:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204188.508466:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204188.508474:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204188.508483:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543396864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204188.508493:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204188.508498:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204188.508502:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543396864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204188.508523:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204188.508526:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204188.508529:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204188.508530:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204188.508532:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204188.508534:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204188.508537:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25874 00000400:00000200:0.0:1788204188.508538:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204188.508540:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204188.508542:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204188.508543:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508545:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508547:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508548:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204188.508549:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204188.508551:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204188.508553:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204188.508555:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204188.508557:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204188.508559:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204188.508561:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508562:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508563:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508564:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204188.508565:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204188.508567:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508569:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204188.508570:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204188.508579:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204188.508582:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508585:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204188.508589:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204188.508592:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508594:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204188.508598:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204188.508602:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204188.508606:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204188.508608:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508611:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204188.508613:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204188.508615:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204188.508620:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204188.508623:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204188.508630:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543396992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204188.508638:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204188.508641:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204188.508646:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543396992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204188.508657:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204188.508661:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204188.508663:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204188.508665:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204188.508667:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204188.508673:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204188.508677:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25874 00000400:00000200:0.0:1788204188.508679:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204188.508681:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204188.508682:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204188.508684:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508685:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508688:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508691:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204188.508696:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204188.508697:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204188.508699:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204188.508700:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204188.508702:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204188.508704:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204188.508707:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508711:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508712:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508713:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204188.508715:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204188.508717:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508718:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204188.508719:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204188.508721:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204188.508722:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508724:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204188.508725:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204188.508726:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508727:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204188.508730:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204188.508733:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204188.508734:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204188.508735:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508736:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204188.508737:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204188.508738:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204188.508741:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204188.508743:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204188.508749:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543397120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204188.508755:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204188.508757:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204188.508762:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543397120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204188.508773:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204188.508775:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204188.508778:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204188.508780:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204188.508782:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204188.508787:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204188.508791:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25874 00000400:00000200:0.0:1788204188.508797:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204188.508801:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204188.508805:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204188.508809:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508814:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508815:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508816:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204188.508817:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204188.508820:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204188.508822:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204188.508824:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204188.508827:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204188.508828:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204188.508830:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508831:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508832:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508833:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204188.508852:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204188.508858:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508862:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204188.508864:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204188.508866:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204188.508868:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508871:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204188.508873:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204188.508876:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508878:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204188.508881:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204188.508885:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204188.508890:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204188.508892:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.508894:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204188.508896:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204188.508899:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204188.508903:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204188.508908:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204188.508912:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543397248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204188.508926:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204188.508929:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204188.508935:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543397248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204188.508941:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204188.508943:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204188.508945:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204188.508946:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204188.508948:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204188.508953:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204188.508955:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25874 00000400:00000200:0.0:1788204188.508961:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204188.508965:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204188.508969:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204188.508971:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508974:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.508977:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.508980:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204188.508983:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204188.508985:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000800:00000200:1.1:1788204188.508985:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:0.0:1788204188.508987:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204188.508989:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204188.508992:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204188.508993:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000800:00000200:1.1:1788204188.508993:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000100:00000040:0.0:1788204188.508999:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.509001:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204188.509006:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204188.509010:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204188.509012:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204188.509017:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204188.509019:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204188.509020:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204188.509021:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204188.509022:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.509024:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204188.509026:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204188.509030:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.509031:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204188.509035:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204188.509039:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204188.509041:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204188.509042:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204188.509043:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204188.509043:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204188.509044:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204188.509046:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204188.509048:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204188.509052:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543397376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204188.509059:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204188.509061:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204188.509065:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543397376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204188.509073:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204188.509080:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25885) 00000800:00000200:1.1:1788204188.509378:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204188.509380:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204188.509557:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204188.509559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204188.509739:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204188.509741:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204188.509926:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204188.509928:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00010000:00000001:0.0:1788204189.340205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204189.340210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204189.340218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204189.340221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204189.340225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204189.340229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204189.340232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204189.340234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204189.340288:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204189.340291:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204189.340294:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204189.852182:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320548614, at 4320548672 00000100:00000010:2.0:1788204189.852194:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16478a0. 00010000:00000001:0.0:1788204190.364189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204190.364193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204190.364200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204190.364203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204190.364210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204190.364213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204190.364219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204190.364222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204190.364284:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204190.364288:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204190.364291:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204191.388192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204191.388198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204191.388212:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204191.388216:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204191.388220:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388224:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204191.388228:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388236:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204191.388240:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204191.388241:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204191.388242:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388244:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204191.388244:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388245:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388247:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204191.388250:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204191.388251:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204191.388252:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388254:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204191.388255:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388256:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204191.388258:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204191.388313:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204191.388317:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204191.388320:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204192.412243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204192.412246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204192.412249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204192.412250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204192.412252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204192.412253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204192.412255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204192.412255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204192.412315:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204192.412319:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204192.412322:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204193.436203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204193.436207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204193.436213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204193.436215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204193.436219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204193.436222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204193.436227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204193.436229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204193.436291:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204193.436294:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204193.436296:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204193.628565:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204193.628577:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204193.628579:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204193.628583:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204193.628587:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204193.628593:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25880 00000400:00000200:0.0:1788204193.628604:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204193.628610:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204193.628613:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204193.628617:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628620:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.628623:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628625:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204193.628628:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204193.628630:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204193.628634:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204193.628637:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204193.628640:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204193.628642:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204193.628645:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628647:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.628649:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628650:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204193.628653:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204193.628656:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628658:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204193.628661:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204193.628664:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204193.628667:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.628669:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204193.628671:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204193.628679:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204193.628683:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204193.628688:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204193.628689:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.628692:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204193.628695:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204193.628698:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204193.628705:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788204193.628711:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204193.628721:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543397504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204193.628731:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204193.628736:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204193.628740:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543397504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204193.628777:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204193.628780:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204193.628784:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204193.628785:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204193.628787:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204193.628790:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204193.628792:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25880 00000400:00000200:0.0:1788204193.628796:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204193.628799:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204193.628801:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204193.628804:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628806:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.628808:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628810:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204193.628812:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204193.628814:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204193.628815:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204193.628818:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204193.628820:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204193.628822:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204193.628824:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628826:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.628828:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628831:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204193.628833:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204193.628835:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628837:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204193.628839:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204193.628845:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204193.628847:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.628848:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204193.628852:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204193.628855:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.628858:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204193.628861:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204193.628863:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204193.628866:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204193.628867:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.628869:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204193.628871:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204193.628872:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204193.628877:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788204193.628881:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204193.628885:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543397632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204193.628893:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204193.628894:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204193.628898:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543397632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204193.628906:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204193.628909:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204193.628912:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204193.628913:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204193.628914:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204193.628917:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204193.628919:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25880 00000400:00000200:0.0:1788204193.628923:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204193.628926:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204193.628929:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204193.628931:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628933:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.628935:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628938:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204193.628940:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204193.628941:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204193.628943:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204193.628945:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204193.628946:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204193.628948:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204193.628951:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628952:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.628954:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628956:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204193.628958:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204193.628960:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204193.628962:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204193.628964:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204193.628967:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204193.628968:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.628970:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204193.628972:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204193.628974:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.628976:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204193.628979:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204193.628981:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204193.628983:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204193.628984:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.628986:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204193.628987:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204193.628989:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204193.628992:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788204193.628995:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204193.629000:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543397760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204193.629013:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204193.629015:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204193.629019:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543397760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204193.629026:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204193.629029:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204193.629032:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204193.629034:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204193.629035:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204193.629038:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204193.629041:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25880 00000400:00000200:0.0:1788204193.629044:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204193.629047:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204193.629049:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204193.629052:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629054:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.629056:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629058:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204193.629060:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204193.629061:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204193.629063:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204193.629065:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204193.629067:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204193.629069:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204193.629072:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629074:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.629076:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629078:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204193.629080:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204193.629083:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629085:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204193.629087:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204193.629108:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204193.629111:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.629113:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204193.629115:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204193.629117:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.629119:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204193.629122:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204193.629125:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204193.629127:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204193.629128:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.629130:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204193.629131:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204193.629133:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204193.629136:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204193.629140:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204193.629144:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543397888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204193.629151:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204193.629153:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204193.629156:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543397888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204193.629164:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204193.629166:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204193.629169:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204193.629171:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204193.629172:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204193.629174:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204193.629177:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25880 00000400:00000200:0.0:1788204193.629181:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204193.629183:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204193.629185:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204193.629188:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629190:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.629192:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629194:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204193.629196:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204193.629197:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204193.629199:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204193.629200:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204193.629202:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204193.629204:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204193.629207:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629209:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204193.629211:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629213:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204193.629215:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204193.629218:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204193.629220:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204193.629222:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204193.629224:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204193.629226:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.629228:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204193.629230:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204193.629232:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.629234:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204193.629237:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204193.629239:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204193.629242:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204193.629243:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204193.629245:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204193.629246:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204193.629248:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204193.629251:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204193.629254:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204193.629258:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543398016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204193.629265:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204193.629267:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204193.629270:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543398016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204193.629277:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204193.629282:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25890) 00000800:00000200:1.1:1788204193.629523:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204193.629526:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204193.629869:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204193.629872:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204193.630190:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204193.630192:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204193.630464:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204193.630466:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204193.630673:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204193.630676:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00010000:00000001:0.0:1788204194.460226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204194.460232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204194.460243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204194.460249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204194.460257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204194.460262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204194.460273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204194.460275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204194.460402:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204194.460406:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204194.460408:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204194.972198:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320553734, at 4320553792 00000100:00000010:2.0:1788204194.972214:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00010000:00000001:0.0:1788204195.484205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204195.484212:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204195.484216:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204195.484220:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204195.484227:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204195.484232:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204195.484238:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204195.484243:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204195.484252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204195.484255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204195.484258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204195.484259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204195.484264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204195.484266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204195.484322:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204195.484327:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204195.484330:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204196.508198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204196.508204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204196.508213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204196.508218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204196.508225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204196.508229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204196.508236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204196.508240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204196.508309:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204196.508313:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204196.508316:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204197.532185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204197.532190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204197.532197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204197.532200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204197.532205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204197.532208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204197.532211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204197.532213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204197.532270:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204197.532274:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204197.532278:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204198.556187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204198.556191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204198.556198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204198.556202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204198.556205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204198.556207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204198.556212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204198.556214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204198.556280:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204198.556284:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204198.556287:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204198.748192:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204198.748202:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204198.748204:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204198.748207:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204198.748211:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204198.748219:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25885 00000400:00000200:2.0:1788204198.748228:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204198.748234:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204198.748238:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204198.748241:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748244:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.748247:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748249:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204198.748261:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204198.748263:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204198.748267:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204198.748271:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204198.748273:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204198.748276:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204198.748278:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748280:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.748282:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748284:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204198.748287:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204198.748290:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748292:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204198.748296:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204198.748299:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204198.748302:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748304:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204198.748306:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204198.748315:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204198.748318:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204198.748322:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204198.748324:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748327:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204198.748329:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204198.748334:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204198.748340:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204198.748346:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204198.748355:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543398144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204198.748365:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204198.748369:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204198.748374:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543398144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204198.748393:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204198.748396:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204198.748400:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204198.748402:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204198.748403:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204198.748406:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204198.748408:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25885 00000400:00000200:2.0:1788204198.748415:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204198.748418:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204198.748420:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204198.748423:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748425:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.748427:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748429:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204198.748431:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204198.748433:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204198.748434:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204198.748436:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204198.748438:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204198.748440:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204198.748443:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748445:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.748447:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748448:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204198.748450:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204198.748452:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748455:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204198.748457:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204198.748463:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204198.748465:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748468:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204198.748474:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204198.748479:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748482:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204198.748486:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204198.748491:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204198.748496:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204198.748498:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748502:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204198.748504:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204198.748508:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204198.748512:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204198.748518:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204198.748526:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543398272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204198.748540:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204198.748543:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204198.748550:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543398272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204198.748563:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204198.748566:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204198.748572:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204198.748575:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204198.748577:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204198.748581:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204198.748586:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25885 00000400:00000200:2.0:1788204198.748592:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204198.748595:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204198.748600:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204198.748604:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748610:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.748614:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748617:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204198.748621:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204198.748623:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204198.748626:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204198.748629:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204198.748641:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204198.748646:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204198.748653:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748702:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.748707:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748712:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204198.748717:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204198.748722:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748726:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204198.748730:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204198.748734:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204198.748738:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748742:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204198.748744:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204198.748749:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748752:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204198.748757:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204198.748761:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204198.748772:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204198.748774:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748777:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204198.748779:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204198.748782:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204198.748787:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204198.748792:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204198.748800:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543398400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1788204198.748807:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204198.748816:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1788204198.748820:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204198.748824:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204198.748830:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543398400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204198.748842:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204198.748846:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204198.748853:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204198.748855:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204198.748857:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204198.748863:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204198.748868:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25885 00000400:00000200:2.0:1788204198.748875:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204198.748882:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204198.748886:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204198.748892:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748895:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.748901:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748904:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204198.748908:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204198.748910:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204198.748913:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204198.748916:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204198.748920:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204198.748925:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204198.748930:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748932:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.748935:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748937:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204198.748941:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204198.748952:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204198.748955:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204198.748957:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204198.748962:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204198.748965:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748967:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204198.748970:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204198.748974:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.748976:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204198.748984:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204198.748988:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204198.748992:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204198.749000:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.749004:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204198.749007:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204198.749011:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204198.749016:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204198.749021:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204198.749027:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543398528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204198.749053:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204198.749058:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204198.749064:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543398528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204198.749077:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204198.749080:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204198.749083:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204198.749085:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204198.749087:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204198.749117:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204198.749121:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25885 00000400:00000200:2.0:1788204198.749125:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204198.749132:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204198.749136:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204198.749141:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.749146:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.749149:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.749153:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204198.749158:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204198.749160:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204198.749162:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204198.749166:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204198.749169:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204198.749173:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204198.749179:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.749182:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204198.749187:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204198.749193:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204198.749198:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204198.749203:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204198.749207:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204198.749211:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204198.749215:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204198.749220:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.749226:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204198.749230:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204198.749235:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.749239:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204198.749245:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204198.749250:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204198.749255:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204198.749258:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204198.749261:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204198.749264:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204198.749268:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204198.749274:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204198.749281:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204198.749289:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543398656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204198.749311:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204198.749316:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204198.749321:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543398656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204198.749335:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204198.749342:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25895) 00000800:00000200:1.1:1788204198.749528:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204198.749531:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204198.749751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204198.749753:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204198.749976:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204198.749978:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204198.750194:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204198.750196:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00010000:00000001:0.0:1788204199.580162:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204199.580167:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204199.580172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204199.580174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204199.580179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204199.580181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204199.580187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204199.580190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204199.580344:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204199.580347:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204199.580349:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204200.092189:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320558853, at 4320558912 00000100:00000010:2.0:1788204200.092202:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647ba0. 00010000:00000001:0.0:1788204200.604193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204200.604199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204200.604213:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204200.604218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204200.604224:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604230:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204200.604234:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604237:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604242:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204200.604247:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204200.604248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204200.604249:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604250:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204200.604251:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604252:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604254:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204200.604258:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204200.604259:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204200.604260:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204200.604261:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604262:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204200.604263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204200.604319:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204200.604322:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204200.604325:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204201.628236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204201.628242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204201.628249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204201.628253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204201.628259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204201.628263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204201.628270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204201.628273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204201.628376:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204201.628384:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204201.628397:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204202.652263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204202.652269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204202.652275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204202.652277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204202.652283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204202.652284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204202.652294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204202.652295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204202.652366:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204202.652371:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204202.652374:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204203.676236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204203.676248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204203.676265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204203.676270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204203.676275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204203.676280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204203.676285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204203.676289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204203.676364:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204203.676368:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204203.676372:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204203.868981:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204203.868989:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204203.870060:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204203.870065:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204203.870470:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204203.870473:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204203.870870:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204203.870873:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204203.871154:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204203.871155:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00010000:00000001:0.0:1788204204.700178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204204.700184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204204.700190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204204.700192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204204.700198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204204.700201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204204.700206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204204.700209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204204.700263:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204204.700266:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204204.700268:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204205.212209:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320563974, at 4320564032 00000100:00000010:2.0:1788204205.212228:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c00. 00010000:00000001:0.0:1788204205.724202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204205.724209:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204205.724213:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204205.724219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204205.724225:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204205.724230:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204205.724235:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204205.724240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204205.724246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204205.724249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204205.724253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204205.724254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204205.724257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204205.724259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204205.724311:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204205.724315:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204205.724318:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204206.748195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204206.748200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204206.748209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204206.748214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204206.748221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204206.748226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204206.748232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204206.748237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204206.748303:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204206.748307:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204206.748310:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204207.772238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204207.772244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204207.772253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204207.772256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204207.772263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204207.772266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204207.772272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204207.772277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204207.772342:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204207.772345:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204207.772348:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204208.796227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204208.796235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204208.796245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204208.796248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204208.796263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204208.796269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204208.796275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204208.796278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204208.796343:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204208.796347:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204208.796349:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204208.988932:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204208.988937:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204208.989758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204208.989760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204208.989972:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204208.989974:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204208.990178:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204208.990180:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204208.990361:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204208.990363:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1788204209.820227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204209.820233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204209.820242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204209.820249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204209.820257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204209.820270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204209.820284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204209.820290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204209.820355:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204209.820360:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204209.820363:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204210.332192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320569094, at 4320569152 00000100:00000010:2.0:1788204210.332208:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcea0. 00010000:00000001:0.0:1788204210.844233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204210.844241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204210.844252:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204210.844263:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204210.844267:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844272:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204210.844277:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844284:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844290:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204210.844303:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204210.844306:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204210.844309:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844311:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204210.844314:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844316:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844319:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844323:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204210.844325:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204210.844326:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204210.844327:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844328:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204210.844329:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844331:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204210.844333:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204210.844387:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204210.844390:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204210.844392:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204211.868197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204211.868201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204211.868207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204211.868209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204211.868211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204211.868213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204211.868216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204211.868217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204211.868282:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204211.868286:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204211.868290:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204212.892209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204212.892214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204212.892223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204212.892227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204212.892233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204212.892236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204212.892242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204212.892245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204212.892299:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204212.892302:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204212.892305:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204213.916196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204213.916202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204213.916211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204213.916216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204213.916222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204213.916225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204213.916229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204213.916233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204213.916259:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204213.916262:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204213.916265:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204214.108212:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204214.108220:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204214.108222:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204214.108225:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204214.108228:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204214.108233:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25900 00000400:00000200:2.0:1788204214.108243:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204214.108248:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204214.108250:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204214.108252:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108255:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108256:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108258:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204214.108260:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204214.108262:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204214.108264:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204214.108267:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204214.108268:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204214.108270:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204214.108272:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108273:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108274:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108275:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204214.108277:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204214.108279:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108280:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204214.108282:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204214.108284:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204214.108286:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108288:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204214.108290:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204214.108297:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204214.108300:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204214.108302:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204214.108303:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108305:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204214.108307:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204214.108310:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204214.108316:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204214.108321:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204214.108328:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543400064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204214.108334:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204214.108338:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204214.108341:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543400064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204214.108357:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204214.108359:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204214.108362:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204214.108363:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204214.108364:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204214.108366:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204214.108367:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25900 00000400:00000200:2.0:1788204214.108369:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204214.108371:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204214.108372:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204214.108373:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108375:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108376:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108378:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204214.108379:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204214.108380:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204214.108382:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204214.108383:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204214.108384:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204214.108385:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204214.108387:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108388:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108390:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108391:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204214.108408:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204214.108410:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108411:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204214.108412:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204214.108417:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204214.108420:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108422:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204214.108424:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204214.108428:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108431:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204214.108433:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204214.108436:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204214.108438:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204214.108440:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108443:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204214.108445:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204214.108448:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204214.108453:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204214.108458:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204214.108465:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543400192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204214.108476:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204214.108483:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204214.108491:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543400192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204214.108506:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204214.108510:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204214.108521:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204214.108525:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204214.108529:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204214.108534:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204214.108537:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25900 00000400:00000200:2.0:1788204214.108542:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204214.108547:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204214.108550:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204214.108556:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108561:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108566:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108571:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204214.108576:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204214.108579:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204214.108582:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204214.108586:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204214.108591:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204214.108594:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204214.108599:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108602:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108604:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108605:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204214.108607:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204214.108611:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108615:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204214.108616:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204214.108619:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204214.108620:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108623:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204214.108625:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204214.108628:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108629:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204214.108633:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204214.108638:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204214.108643:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204214.108647:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108652:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204214.108654:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204214.108658:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204214.108663:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204214.108669:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204214.108678:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543400320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204214.108701:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204214.108704:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204214.108709:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543400320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204214.108717:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204214.108720:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204214.108724:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204214.108728:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204214.108730:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204214.108736:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.1:1788204214.108738:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204214.108741:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25900 00000800:00000200:3.1:1788204214.108743:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000400:00000200:2.0:1788204214.108748:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204214.108756:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204214.108761:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204214.108764:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108769:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108772:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108776:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204214.108781:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204214.108782:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204214.108784:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204214.108786:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204214.108789:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204214.108793:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204214.108795:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108798:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108802:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108804:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204214.108808:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204214.108813:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108815:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204214.108816:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204214.108818:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204214.108819:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108820:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204214.108821:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204214.108825:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108827:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204214.108830:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204214.108833:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204214.108834:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204214.108836:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.108840:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204214.108842:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204214.108843:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204214.108846:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204214.108850:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204214.108855:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543400448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204214.108871:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204214.108875:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204214.108880:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543400448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204214.108886:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204214.108889:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204214.108892:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204214.108893:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204214.108894:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204214.108898:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204214.108902:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25900 00000400:00000200:2.0:1788204214.108909:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204214.108913:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204214.108917:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204214.108921:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108925:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108928:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108934:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204214.108939:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204214.108942:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204214.108946:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204214.108948:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204214.108952:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204214.108957:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204214.108961:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108964:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204214.108969:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108982:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204214.108987:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204214.108992:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204214.108998:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204214.109004:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204214.109007:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204214.109010:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.109012:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204214.109015:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204214.109022:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.109026:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204214.109033:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204214.109039:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204214.109044:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204214.109049:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204214.109053:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204214.109057:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204214.109068:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204214.109077:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204214.109084:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204214.109116:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543400576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204214.109124:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204214.109127:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204214.109132:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543400576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204214.109149:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204214.109156:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25910) 00000800:00000200:3.1:1788204214.109365:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204214.109368:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204214.109659:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204214.109661:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204214.109968:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204214.109971:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204214.110449:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204214.110452:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788204214.940190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204214.940195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204214.940203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204214.940206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204214.940209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204214.940210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204214.940213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204214.940215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204214.940274:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204214.940278:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204214.940281:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204215.452166:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320574213, at 4320574272 00000100:00000010:2.0:1788204215.452179:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc7e0. 00010000:00000001:0.0:1788204215.964221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204215.964231:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204215.964239:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204215.964245:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204215.964259:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204215.964268:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204215.964280:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204215.964286:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204215.964294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204215.964298:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204215.964305:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204215.964307:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204215.964311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204215.964313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204215.964384:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204215.964389:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204215.964392:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204216.988211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204216.988219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204216.988229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204216.988233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204216.988243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204216.988247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204216.988264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204216.988268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204216.989137:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204216.989141:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204216.989145:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204218.012233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204218.012241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204218.012250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204218.012252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204218.012256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204218.012258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204218.012262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204218.012264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204218.012348:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204218.012354:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204218.012360:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204219.036191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204219.036196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204219.036205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204219.036209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204219.036212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204219.036215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204219.036219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204219.036222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204219.036285:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204219.036290:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204219.036293:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204219.228887:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204219.228894:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204219.229542:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204219.229545:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204219.229760:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204219.229761:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204219.230039:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204219.230042:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204219.230510:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204219.230512:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204220.060252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204220.060262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204220.060270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204220.060271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204220.060274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204220.060276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204220.060278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204220.060280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204220.060339:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204220.060342:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204220.060345:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204220.572198:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320579333, at 4320579392 00000100:00000010:2.0:1788204220.572216:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00010000:00000001:0.0:1788204221.084181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204221.084186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204221.084197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204221.084202:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204221.084207:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084212:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204221.084215:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084218:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084223:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204221.084229:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204221.084230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204221.084231:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084232:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204221.084233:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084234:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084236:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204221.084239:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204221.084241:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204221.084242:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084243:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204221.084244:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084245:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204221.084246:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204221.084308:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204221.084311:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204221.084313:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204222.108178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204222.108182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204222.108189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204222.108193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204222.108199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204222.108202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204222.108207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204222.108211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204222.108277:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204222.108281:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204222.108284:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204223.132184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204223.132189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204223.132195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204223.132199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204223.132203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204223.132206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204223.132211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204223.132214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204223.132277:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204223.132282:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204223.132285:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204224.156190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204224.156195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204224.156201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204224.156212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204224.156215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204224.156216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204224.156233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204224.156235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204224.156291:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204224.156296:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204224.156299:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204224.348240:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204224.348250:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204224.348253:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204224.348256:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204224.348260:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204224.348267:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25910 00000400:00000200:2.0:1788204224.348279:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204224.348287:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204224.348290:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204224.348293:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348296:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.348299:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348301:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204224.348305:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204224.348307:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204224.348310:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204224.348313:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204224.348315:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204224.348318:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204224.348321:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348323:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.348324:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348326:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204224.348329:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204224.348332:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348335:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204224.348337:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204224.348340:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204224.348343:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348345:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204224.348349:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204224.348358:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204224.348361:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204224.348364:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204224.348366:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348369:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204224.348372:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204224.348376:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204224.348389:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204224.348396:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204224.348406:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543401344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204224.348416:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204224.348420:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204224.348424:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543401344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204224.348445:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204224.348448:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204224.348451:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204224.348453:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204224.348455:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204224.348457:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204224.348460:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25910 00000400:00000200:2.0:1788204224.348463:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204224.348467:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204224.348469:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204224.348472:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348475:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.348477:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348479:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204224.348482:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204224.348483:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204224.348485:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204224.348492:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204224.348496:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204224.348500:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204224.348506:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348508:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.348512:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348516:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204224.348519:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204224.348521:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348523:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204224.348527:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204224.348533:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204224.348537:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348542:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204224.348547:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204224.348553:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348557:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204224.348562:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204224.348567:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204224.348572:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204224.348575:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348579:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204224.348580:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204224.348583:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204224.348590:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204224.348596:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204224.348603:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543401472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204224.348616:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204224.348620:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204224.348625:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543401472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204224.348636:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204224.348640:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204224.348645:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204224.348647:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204224.348651:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204224.348656:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204224.348661:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25910 00000400:00000200:2.0:1788204224.348667:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204224.348673:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204224.348677:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204224.348682:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348688:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.348692:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348696:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204224.348700:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204224.348703:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204224.348706:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204224.348708:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204224.348713:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204224.348717:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204224.348721:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348723:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.348727:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348730:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204224.348733:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204224.348737:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348743:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204224.348745:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204224.348749:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204224.348752:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348755:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204224.348760:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204224.348764:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348767:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204224.348772:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204224.348776:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204224.348780:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204224.348782:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348786:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204224.348788:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204224.348791:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204224.348797:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204224.348800:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204224.348809:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543401600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204224.348822:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204224.348825:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204224.348831:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543401600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204224.348847:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204224.348852:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1788204224.348858:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204224.348859:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204224.348862:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204224.348865:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1788204224.348866:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788204224.348869:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204224.348877:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25910 00000400:00000200:2.0:1788204224.348884:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204224.348889:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204224.348892:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204224.348897:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348901:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.348906:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348910:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204224.348914:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204224.348917:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204224.348920:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204224.348923:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204224.348925:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204224.348929:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204224.348934:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348939:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.348942:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348946:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204224.348951:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204224.348956:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204224.348960:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204224.348964:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204224.348968:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204224.348973:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348977:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204224.348981:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204224.348987:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.348990:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204224.348993:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204224.348997:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204224.349001:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204224.349003:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.349006:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204224.349009:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204224.349011:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204224.349016:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204224.349021:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204224.349027:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543401728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204224.349039:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204224.349043:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204224.349049:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543401728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204224.349064:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204224.349068:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204224.349072:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204224.349076:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204224.349078:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204224.349084:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204224.349088:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25910 00000400:00000200:2.0:1788204224.349132:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204224.349136:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204224.349141:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204224.349147:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.349151:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.349155:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.349159:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204224.349164:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204224.349167:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204224.349171:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204224.349174:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204224.349179:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204224.349184:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204224.349189:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.349193:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204224.349197:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204224.349201:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204224.349205:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204224.349211:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204224.349234:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204224.349237:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204224.349241:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204224.349245:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.349250:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204224.349253:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204224.349257:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.349260:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204224.349265:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204224.349270:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204224.349275:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204224.349277:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204224.349281:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204224.349283:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204224.349286:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204224.349291:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204224.349298:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204224.349306:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543401856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204224.349322:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204224.349327:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204224.349334:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543401856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204224.349359:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204224.349365:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25920) 00000800:00000200:1.1:1788204224.349549:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204224.349551:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204224.349823:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204224.349825:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204224.350109:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204224.350112:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204224.350401:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204224.350403:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204225.180216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204225.180220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204225.180225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204225.180228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204225.180232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204225.180235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204225.180239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204225.180243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204225.180321:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204225.180326:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204225.180329:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204225.692177:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320584453, at 4320584512 00000100:00000010:2.0:1788204225.692205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00010000:00000001:0.0:1788204226.204161:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204226.204171:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204226.204176:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204226.204179:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204226.204193:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204226.204198:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204226.204201:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204226.204207:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204226.204216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204226.204220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204226.204226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204226.204230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204226.204237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204226.204238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204226.204283:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204226.204287:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204226.204289:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204227.228210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204227.228215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204227.228223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204227.228225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204227.228230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204227.228232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204227.228240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204227.228249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204227.228317:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204227.228321:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204227.228324:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204228.252217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204228.252223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204228.252232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204228.252235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204228.252241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204228.252244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204228.252250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204228.252252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204228.252313:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204228.252317:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204228.252320:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204229.276233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204229.276240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204229.276247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204229.276257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204229.276264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204229.276268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204229.276275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204229.276278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204229.276344:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204229.276348:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204229.276351:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204229.468751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204229.468755:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204229.469366:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204229.469369:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204229.469588:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204229.469589:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204229.469802:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204229.469804:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204229.470019:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204229.470020:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1788204230.300206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204230.300210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204230.300215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204230.300218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204230.300226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204230.300228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204230.300235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204230.300238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204230.300289:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204230.300293:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204230.300295:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204230.812202:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320589573, at 4320589632 00000100:00000010:2.0:1788204230.812216:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16474e0. 00010000:00000001:0.0:1788204231.324193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204231.324199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204231.324211:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204231.324216:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204231.324219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324223:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204231.324226:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324228:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204231.324235:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204231.324237:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204231.324239:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324240:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204231.324242:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324243:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324245:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204231.324250:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204231.324251:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204231.324252:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324253:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204231.324254:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324256:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204231.324258:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204231.324309:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204231.324314:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204231.324317:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204232.348234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204232.348240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204232.348247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204232.348253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204232.348257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204232.348262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204232.348266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204232.348272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204232.348343:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204232.348349:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204232.348353:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204233.372197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204233.372202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204233.372209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204233.372212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204233.372218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204233.372220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204233.372224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204233.372227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204233.372280:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204233.372283:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204233.372286:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204234.396162:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204234.396166:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204234.396170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204234.396172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204234.396176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204234.396179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204234.396182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204234.396186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204234.396245:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204234.396249:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204234.396253:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204234.588570:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204234.588573:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204234.588934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204234.588936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204234.589169:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204234.589170:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204234.589327:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204234.589329:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204234.589477:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204234.589478:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00010000:00000001:0.0:1788204235.420172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204235.420178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204235.420186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204235.420190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204235.420195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204235.420199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204235.420204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204235.420208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204235.420261:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204235.420265:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204235.420268:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204235.932267:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320594693, at 4320594752 00000100:00000010:2.0:1788204235.932280:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00010000:00000001:0.0:1788204236.444172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204236.444180:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204236.444184:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204236.444187:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204236.444192:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204236.444196:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204236.444199:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204236.444205:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204236.444212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204236.444214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204236.444216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204236.444217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204236.444220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204236.444221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204236.444257:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204236.444260:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204236.444263:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204237.468196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204237.468202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204237.468210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204237.468212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204237.468217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204237.468221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204237.468227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204237.468231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204237.468282:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204237.468285:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204237.468287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204238.492185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204238.492194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204238.492198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204238.492202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204238.492205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204238.492209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204238.492213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204238.492216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204238.492270:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204238.492274:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204238.492277:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204239.516200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204239.516204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204239.516211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204239.516214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204239.516220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204239.516224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204239.516227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204239.516229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204239.516281:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204239.516285:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204239.516288:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204239.708791:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204239.709237:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204239.709420:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204239.709555:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204239.709694:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204240.540204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204240.540211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204240.540225:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204240.540228:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204240.540234:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204240.540244:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540247:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204240.540255:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204240.540256:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204240.540257:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540259:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204240.540261:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540263:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540265:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204240.540271:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204240.540273:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204240.540275:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540277:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204240.540278:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540279:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204240.540281:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204240.540343:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204240.540348:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204240.540351:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204241.052187:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320599813, at 4320599872 00000100:00000010:2.0:1788204241.052204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00010000:00000001:0.0:1788204241.564200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204241.564204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204241.564213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204241.564216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204241.564222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204241.564224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204241.564228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204241.564231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204241.564315:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204241.564322:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204241.564326:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204242.588188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204242.588194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204242.588199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204242.588202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204242.588207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204242.588208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204242.588210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204242.588212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204242.588275:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204242.588279:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204242.588283:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788204242.780429:0:256123:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:0.0:1788204242.780473:0:256123:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204242.780498:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204242.780504:0:256123:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:0.0:1788204242.780509:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204242.780513:0:256123:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:0.0:1788204242.780518:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204242.780522:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:0.0:1788204242.780527:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204242.780532:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204242.780539:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6f68 00000400:00000010:0.0:1788204242.780542:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6f68. 00000100:00000001:0.0:1788204242.780548:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204242.780559:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a680 x1875055543403264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780589:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204242.780593:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543403264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780600:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204242.780602:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204242.780613:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204242.780615:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97a00. 00000400:00000200:0.0:1788204242.780618:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204242.780621:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204242.780625:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a67f8 00000400:00000010:0.0:1788204242.780626:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a67f8. 00000100:00000001:0.0:1788204242.780629:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204242.780633:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39f80 x1875055543403392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780641:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204242.780645:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543403392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780650:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204242.780652:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204242.780656:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204242.780658:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:0.0:1788204242.780661:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204242.780664:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204242.780666:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6110 00000400:00000010:0.0:1788204242.780668:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6110. 00000100:00000001:0.0:1788204242.780671:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204242.780675:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3b480 x1875055543403520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780682:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204242.780685:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543403520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780692:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204242.780693:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204242.780698:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204242.780701:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000400:00000200:0.0:1788204242.780703:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204242.780706:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204242.780709:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6ee0 00000400:00000010:0.0:1788204242.780711:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6ee0. 00000100:00000001:0.0:1788204242.780713:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204242.780717:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39c00 x1875055543403648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780724:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204242.780727:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543403648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780734:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204242.780735:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204242.780740:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204242.780742:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000400:00000200:0.0:1788204242.780745:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204242.780748:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204242.780751:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a66e8 00000400:00000010:0.0:1788204242.780752:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a66e8. 00000100:00000001:0.0:1788204242.780755:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204242.780759:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38000 x1875055543403776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780772:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204242.780775:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543403776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204264 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204242.780781:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204242.780782:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1788204243.612162:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204243.612168:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204243.612177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204243.612180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204243.612186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204243.612189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204243.612193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204243.612196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204243.612258:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204243.612262:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204243.612265:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204244.636210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204244.636216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204244.636222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204244.636228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204244.636230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204244.636232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204244.636236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204244.636244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204244.636317:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204244.636321:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204244.636326:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204244.828417:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204244.828428:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204244.828431:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204244.828434:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204244.828438:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204244.828446:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25931 00000400:00000200:2.0:1788204244.828456:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204244.828463:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204244.828465:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204244.828469:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828471:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.828473:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828475:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204244.828480:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204244.828482:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204244.828486:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204244.828489:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204244.828492:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204244.828494:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204244.828498:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828499:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.828505:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828508:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204244.828513:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204244.828516:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828519:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204244.828523:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204244.828527:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204244.828530:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.828533:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204244.828536:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204244.828542:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204244.828546:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204244.828549:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204244.828551:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.828554:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204244.828557:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204244.828561:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204244.828568:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204244.828576:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204244.828586:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543403904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204244.828596:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204244.828601:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204244.828605:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543403904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204244.828634:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204244.828638:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204244.828641:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204244.828643:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204244.828645:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204244.828647:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204244.828651:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25931 00000400:00000200:2.0:1788204244.828655:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204244.828659:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204244.828661:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204244.828664:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828666:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.828668:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828670:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204244.828672:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204244.828673:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204244.828675:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204244.828677:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204244.828680:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204244.828682:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204244.828685:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828687:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.828689:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828691:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204244.828694:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204244.828696:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828699:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204244.828700:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204244.828706:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204244.828708:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.828710:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204244.828714:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204244.828717:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.828719:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204244.828722:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204244.828724:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204244.828727:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204244.828728:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.828730:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204244.828732:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204244.828734:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204244.828738:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204244.828741:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204244.828745:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543404032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204244.828752:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204244.828754:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204244.828758:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543404032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204244.828766:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204244.828769:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204244.828772:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204244.828774:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204244.828775:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204244.828778:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204244.828781:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25931 00000400:00000200:2.0:1788204244.828785:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204244.828788:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204244.828790:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204244.828794:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828796:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.828798:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828799:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204244.828801:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204244.828803:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204244.828804:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204244.828806:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204244.828809:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204244.828811:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204244.828814:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828816:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.828818:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828820:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204244.828823:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204244.828825:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204244.828828:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204244.828830:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204244.828833:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204244.828835:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.828837:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204244.828839:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204244.828842:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.828844:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204244.828847:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204244.828849:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204244.828851:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204244.828853:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.828854:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204244.828856:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204244.828857:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204244.829142:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204244.829146:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204244.829473:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204244.829476:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 02000000:00000010:2.0:1788204244.829777:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204244.829783:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204244.829789:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543404160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204244.829795:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204244.829798:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204244.829802:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543404160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204244.829820:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204244.829823:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204244.829826:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204244.829828:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204244.829829:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204244.829833:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204244.829835:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25931 00000400:00000200:2.0:1788204244.829840:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204244.829843:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204244.829845:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204244.829848:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.829851:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.829853:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.829855:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204244.829857:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204244.829867:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204244.829869:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204244.829872:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204244.829874:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204244.829876:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204244.829879:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.829881:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.829883:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.829885:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204244.829887:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204244.829890:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204244.829893:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204244.829895:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204244.829899:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204244.829901:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.829903:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204244.829906:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204244.829909:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.829911:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204244.829915:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204244.829918:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204244.829921:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204244.829923:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.829924:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204244.829926:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204244.829928:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204244.829932:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204244.829935:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204244.829940:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543404288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204244.829947:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204244.829949:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204244.829952:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543404288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204244.829959:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204244.829961:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204244.829965:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204244.829966:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204244.829968:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204244.829971:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204244.829973:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25931 00000400:00000200:2.0:1788204244.829977:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204244.829980:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204244.829982:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204244.829985:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.829987:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.829989:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.829991:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204244.829993:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204244.829995:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204244.829997:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204244.829998:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204244.830001:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204244.830003:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204244.830006:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.830008:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204244.830010:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204244.830012:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204244.830014:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204244.830017:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204244.830019:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204244.830021:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204244.830024:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204244.830026:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.830028:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204244.830030:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204244.830032:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.830034:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204244.830037:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204244.830039:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204244.830042:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204244.830043:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204244.830045:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204244.830047:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204244.830048:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204244.830051:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204244.830055:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204244.830059:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543404416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204244.830069:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204244.830071:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204244.830073:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543404416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204244.830078:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204244.830083:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25941) 00000800:00000200:1.1:1788204244.831955:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204244.831958:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204244.832295:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204244.832298:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204244.832956:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204244.832959:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204245.660179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204245.660184:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204245.660187:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204245.660191:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204245.660197:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204245.660201:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204245.660205:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204245.660210:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204245.660218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204245.660227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204245.660235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204245.660237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204245.660240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204245.660241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204245.660287:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204245.660291:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204245.660293:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204246.172193:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320604933, at 4320604992 00000100:00000010:2.0:1788204246.172207:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647660. 00010000:00000001:0.0:1788204246.684203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204246.684209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204246.684216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204246.684220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204246.684228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204246.684232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204246.684236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204246.684239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204246.684300:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204246.684304:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204246.684307:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204247.708335:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204247.708338:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204247.708343:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204247.708344:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204247.708347:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204247.708348:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204247.708351:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204247.708353:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204247.708434:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204247.708437:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204247.708439:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204248.732190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204248.732196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204248.732205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204248.732209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204248.732220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204248.732224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204248.732230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204248.732232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204248.732290:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204248.732293:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204248.732295:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204249.756205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204249.756210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204249.756219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204249.756223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204249.756229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204249.756233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204249.756241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204249.756245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204249.756309:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204249.756313:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204249.756317:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204249.948729:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204249.948736:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204249.949378:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204249.949381:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204249.949556:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204249.949558:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204249.949841:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204249.949843:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204249.950124:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204249.950126:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788204250.780191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204250.780203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204250.780214:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204250.780218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204250.780223:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204250.780235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780238:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780245:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204250.780251:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204250.780253:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204250.780254:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780256:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204250.780257:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780259:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780262:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204250.780268:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204250.780270:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204250.780272:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780274:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204250.780275:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780277:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204250.780278:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204250.780324:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204250.780327:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204250.780329:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204251.292196:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320610053, at 4320610112 00000100:00000010:2.0:1788204251.292209:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647960. 00010000:00000001:0.0:1788204251.804180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204251.804187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204251.804196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204251.804200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204251.804206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204251.804209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204251.804216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204251.804218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204251.804272:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204251.804276:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204251.804279:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204252.828201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204252.828206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204252.828213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204252.828216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204252.828221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204252.828223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204252.828230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204252.828233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204252.828301:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204252.828306:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204252.828309:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204253.852180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204253.852188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204253.852196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204253.852197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204253.852203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204253.852206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204253.852212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204253.852216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204253.852276:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204253.852281:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204253.852284:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204254.876195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204254.876201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204254.876209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204254.876214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204254.876222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204254.876226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204254.876231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204254.876234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204254.876373:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204254.876375:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204254.876377:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204255.068761:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204255.068767:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204255.069472:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204255.069475:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204255.069769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204255.069771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204255.070040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204255.070042:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204255.070440:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204255.070442:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1788204255.900178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204255.900184:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204255.900199:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204255.900204:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204255.900210:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204255.900214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204255.900216:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204255.900219:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204255.900224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204255.900226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204255.900229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204255.900231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204255.900234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204255.900235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204255.900285:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204255.900289:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204255.900292:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204256.412188:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320615173, at 4320615232 00000100:00000010:2.0:1788204256.412204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647420. 00010000:00000001:0.0:1788204256.924207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204256.924212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204256.924220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204256.924224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204256.924230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204256.924233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204256.924238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204256.924242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204256.924422:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204256.924425:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204256.924429:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204257.948187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204257.948193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204257.948203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204257.948207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204257.948211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204257.948223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204257.948231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204257.948234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204257.948293:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204257.948298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204257.948301:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204258.972200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204258.972205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204258.972213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204258.972217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204258.972225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204258.972228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204258.972234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204258.972239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204258.972307:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204258.972311:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204258.972313:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204259.996195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204259.996200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204259.996208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204259.996212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204259.996220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204259.996225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204259.996235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204259.996238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204259.996303:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204259.996307:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204259.996311:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204260.188762:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204260.188803:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204260.189486:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204260.189488:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204260.189755:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204260.189757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204260.190006:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204260.190009:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204260.190270:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204260.190272:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1788204261.020162:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204261.020167:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204261.020180:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204261.020184:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204261.020189:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020195:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204261.020200:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020203:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020205:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204261.020210:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204261.020211:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204261.020213:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020214:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204261.020215:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020216:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204261.020221:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204261.020221:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204261.020222:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020223:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204261.020224:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020225:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204261.020226:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204261.020266:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204261.020270:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204261.020272:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204261.532183:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320620293, at 4320620352 00000100:00000010:2.0:1788204261.532196:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00010000:00000001:0.0:1788204262.044193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204262.044197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204262.044205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204262.044210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204262.044219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204262.044223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204262.044238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204262.044243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204262.044292:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204262.044295:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204262.044297:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204263.068171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204263.068175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204263.068188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204263.068193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204263.068201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204263.068206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204263.068211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204263.068215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204263.068279:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204263.068284:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204263.068287:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204264.092186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204264.092191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204264.092197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204264.092199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204264.092206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204264.092209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204264.092216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204264.092218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204264.092271:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204264.092274:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204264.092276:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204265.116219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204265.116224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204265.116232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204265.116236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204265.116241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204265.116245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204265.116249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204265.116252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204265.116312:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204265.116318:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204265.116323:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204265.308181:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204265.308190:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204265.308192:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204265.308195:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204265.308198:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204265.308204:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25951 00000400:00000200:0.0:1788204265.308211:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204265.308217:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204265.308219:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204265.308222:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308224:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308225:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308227:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204265.308230:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204265.308231:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204265.308234:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204265.308236:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204265.308238:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204265.308240:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204265.308242:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308243:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308244:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308245:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204265.308247:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204265.308249:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308251:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204265.308253:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204265.308256:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204265.308258:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308259:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204265.308261:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204265.308269:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204265.308272:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204265.308275:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204265.308276:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308278:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204265.308280:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204265.308283:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204265.308288:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788204265.308294:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204265.308302:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543406464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204265.308309:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204265.308312:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204265.308315:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543406464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204265.308333:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204265.308335:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204265.308337:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204265.308338:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204265.308339:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204265.308341:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204265.308343:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25951 00000400:00000200:0.0:1788204265.308344:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204265.308346:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204265.308348:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204265.308350:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308351:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308353:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308355:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204265.308357:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204265.308358:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204265.308359:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204265.308361:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204265.308362:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204265.308364:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204265.308367:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308369:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308370:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308373:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204265.308375:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204265.308378:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308381:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204265.308383:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204265.308389:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204265.308393:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308395:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204265.308400:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204265.308404:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308406:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204265.308410:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204265.308413:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204265.308415:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204265.308415:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308416:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204265.308417:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204265.308419:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204265.308421:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204265.308425:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204265.308437:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543406592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204265.308444:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204265.308447:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204265.308451:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543406592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204265.308462:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204265.308465:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204265.308467:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204265.308468:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204265.308471:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204265.308473:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204265.308475:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25951 00000400:00000200:0.0:1788204265.308478:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204265.308483:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204265.308486:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204265.308489:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308492:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308495:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308499:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204265.308502:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204265.308574:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204265.308578:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204265.308580:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204265.308584:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204265.308588:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204265.308594:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308597:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308601:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308612:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204265.308618:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204265.308624:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308630:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204265.308633:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204265.308638:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204265.308642:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308646:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204265.308650:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1788204265.308652:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000008:00000001:0.0:1788204265.308655:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204265.308657:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00000001:0.0:1788204265.308659:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204265.308663:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204265.308666:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204265.308670:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204265.308671:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308672:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204265.308675:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204265.308677:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204265.308682:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204265.308686:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204265.308693:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543406720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204265.308707:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204265.308711:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204265.308721:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543406720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204265.308731:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204265.308734:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204265.308741:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204265.308744:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204265.308747:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204265.308751:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204265.308756:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25951 00000400:00000200:0.0:1788204265.308763:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204265.308770:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204265.308774:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204265.308782:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308786:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308792:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308796:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204265.308801:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204265.308803:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204265.308806:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204265.308809:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204265.308813:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204265.308817:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204265.308829:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308832:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308836:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308840:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204265.308845:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204265.308850:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308858:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204265.308860:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204265.308863:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204265.308865:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308869:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204265.308871:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204265.308875:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308876:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204265.308881:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204265.308885:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204265.308887:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204265.308890:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.308894:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204265.308895:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204265.308897:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204265.308900:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204265.308905:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204265.308913:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543406848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204265.308921:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204265.308925:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204265.308932:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543406848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204265.308944:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204265.308947:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204265.308951:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204265.308953:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204265.308956:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204265.308962:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204265.308967:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25951 00000400:00000200:0.0:1788204265.308971:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204265.308976:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204265.308980:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204265.308985:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308989:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.308993:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.308995:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204265.308998:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204265.309000:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204265.309002:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204265.309005:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204265.309007:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204265.309012:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204265.309016:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.309018:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204265.309021:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204265.309024:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204265.309028:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204265.309031:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204265.309035:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204265.309039:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204265.309041:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204265.309044:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.309046:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204265.309049:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204265.309051:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.309054:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204265.309056:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204265.309059:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204265.309062:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204265.309064:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204265.309066:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204265.309069:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204265.309072:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204265.309075:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204265.309079:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204265.309086:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543406976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204265.309130:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204265.309133:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204265.309140:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543406976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204265.309151:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204265.309158:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25961) 00000800:00000200:1.1:1788204265.309233:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204265.309234:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788204265.309434:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204265.309435:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788204265.309617:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204265.309618:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788204265.309888:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204265.309890:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00010000:00000001:0.0:1788204266.140183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204266.140189:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204266.140192:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204266.140198:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204266.140204:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204266.140208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204266.140211:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204266.140216:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204266.140220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204266.140223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204266.140229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204266.140232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204266.140236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204266.140238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204266.140295:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204266.140298:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204266.140301:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204266.652158:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320625413, at 4320625472 00000100:00000010:2.0:1788204266.652175:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647000. 00010000:00000001:0.0:1788204267.164182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204267.164185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204267.164191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204267.164194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204267.164199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204267.164201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204267.164207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204267.164211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204267.164254:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204267.164257:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204267.164260:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204268.188187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204268.188191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204268.188198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204268.188200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204268.188205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204268.188209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204268.188215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204268.188218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204268.188264:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204268.188267:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204268.188270:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204269.212188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204269.212203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204269.212216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204269.212219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204269.212225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204269.212228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204269.212232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204269.212234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204269.212281:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204269.212284:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204269.212286:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204270.236194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204270.236199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204270.236207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204270.236210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204270.236216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204270.236220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204270.236227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204270.236230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204270.236283:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204270.236287:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204270.236289:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204270.428692:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204270.428697:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204270.429217:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204270.429219:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204270.429378:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204270.429379:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204270.429572:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204270.429574:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204270.429754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204270.429756:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00010000:00000001:0.0:1788204271.260168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204271.260172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204271.260183:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204271.260185:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204271.260190:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260196:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204271.260199:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260202:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260206:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204271.260215:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204271.260217:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204271.260219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260220:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204271.260221:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260222:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260223:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204271.260228:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204271.260229:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204271.260230:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204271.260232:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204271.260234:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204271.260270:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204271.260273:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204271.260276:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204271.772166:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320630533, at 4320630592 00000100:00000010:2.0:1788204271.772179:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc4e0. 00010000:00000001:0.0:1788204272.284197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204272.284201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204272.284208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204272.284211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204272.284215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204272.284218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204272.284223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204272.284227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204272.284278:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204272.284282:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204272.284285:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204273.308173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204273.308178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204273.308187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204273.308192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204273.308200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204273.308202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204273.308206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204273.308208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204273.308256:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204273.308260:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204273.308263:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204274.332191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204274.332195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204274.332201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204274.332204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204274.332208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204274.332211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204274.332214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204274.332217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204275.356188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204275.356194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204275.356203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204275.356207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204275.356211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204275.356214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204275.356219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204275.356223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204275.356275:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204275.356278:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204275.356281:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204275.548213:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204275.548223:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204275.548225:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204275.548229:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204275.548233:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204275.548241:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25961 00000400:00000200:2.0:1788204275.548251:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204275.548257:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204275.548261:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204275.548264:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548268:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.548270:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548272:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204275.548276:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204275.548278:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204275.548281:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204275.548285:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204275.548287:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204275.548290:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204275.548293:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548295:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.548297:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548299:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204275.548302:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204275.548306:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548309:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204275.548311:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204275.548315:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204275.548318:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548321:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204275.548323:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204275.548361:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204275.548365:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204275.548369:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204275.548371:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548374:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204275.548377:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204275.548380:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204275.548389:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204275.548395:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204275.548404:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543407744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204275.548415:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204275.548419:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204275.548423:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543407744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204275.548447:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204275.548451:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204275.548454:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204275.548455:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204275.548464:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204275.548467:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204275.548471:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25961 00000400:00000200:2.0:1788204275.548476:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204275.548479:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204275.548481:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204275.548484:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548486:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.548488:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548490:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204275.548492:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204275.548493:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204275.548495:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204275.548498:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204275.548500:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204275.548502:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204275.548505:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548507:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.548509:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548511:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204275.548513:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204275.548516:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548518:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204275.548520:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204275.548525:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204275.548532:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548534:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204275.548538:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204275.548541:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548543:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204275.548547:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204275.548549:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204275.548552:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204275.548553:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548555:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204275.548556:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204275.548558:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204275.548562:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204275.548566:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204275.548571:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543407872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204275.548588:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204275.548591:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204275.548598:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543407872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204275.548611:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204275.548617:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204275.548627:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204275.548630:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204275.548642:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204275.548648:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204275.548654:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25961 00000400:00000200:2.0:1788204275.548662:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204275.548668:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204275.548672:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204275.548679:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548681:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.548686:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548692:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204275.548694:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204275.548697:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204275.548700:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204275.548701:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204275.548703:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204275.548705:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204275.548708:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548710:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.548712:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548714:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204275.548716:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204275.548719:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548721:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204275.548723:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204275.548725:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204275.548727:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548729:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204275.548731:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204275.548734:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548735:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204275.548738:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204275.548741:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204275.548743:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204275.548744:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548746:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204275.548747:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204275.548749:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204275.548752:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204275.548755:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204275.548759:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543408000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204275.548766:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204275.548768:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204275.548771:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543408000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204275.548778:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204275.548780:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204275.548783:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204275.548785:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204275.548786:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204275.548790:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204275.548794:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25961 00000400:00000200:2.0:1788204275.548801:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204275.548805:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204275.548810:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204275.548813:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548815:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.548820:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548825:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204275.548829:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204275.548831:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204275.548834:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204275.548838:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204275.548843:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204275.548847:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204275.548852:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548858:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.548863:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548866:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204275.548870:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204275.548873:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204275.548874:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204275.548876:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204275.548880:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204275.548884:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548886:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204275.548888:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204275.548893:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548896:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204275.548902:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204275.548905:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204275.548910:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204275.548914:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.548918:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204275.548922:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204275.548925:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204275.548932:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204275.548939:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204275.548958:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543408128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204275.548971:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204275.548974:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204275.548979:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543408128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204275.548989:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204275.548992:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204275.548998:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204275.548999:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204275.549001:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.1:1788204275.549002:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204275.549004:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.1:1788204275.549007:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00080000:2.0:1788204275.549008:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25961 00000400:00000200:2.0:1788204275.549016:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204275.549020:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204275.549022:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204275.549024:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.549025:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.549027:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.549028:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204275.549029:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204275.549030:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204275.549031:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204275.549032:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204275.549033:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204275.549035:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204275.549037:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.549038:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204275.549039:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204275.549040:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204275.549042:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204275.549043:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204275.549044:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204275.549045:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204275.549047:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204275.549048:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.549049:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204275.549050:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204275.549051:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.549052:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204275.549054:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204275.549055:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204275.549057:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204275.549058:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204275.549059:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204275.549060:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204275.549061:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204275.549063:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204275.549065:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204275.549068:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543408256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204275.549073:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204275.549074:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204275.549076:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543408256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204275.549080:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204275.549083:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25972) 00000800:00000200:3.1:1788204275.549505:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204275.549507:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204275.549688:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204275.549690:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204275.549916:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204275.549918:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204275.550073:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204275.550075:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1788204276.380197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204276.380204:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204276.380208:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204276.380212:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204276.380217:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204276.380220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204276.380224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204276.380229:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204276.380235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204276.380283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204276.380286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204276.380287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204276.380289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204276.380290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204276.380360:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204276.380364:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204276.380367:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204276.892190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320635654, at 4320635712 00000100:00000010:2.0:1788204276.892206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc240. 00010000:00000001:0.0:1788204277.404192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204277.404197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204277.404203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204277.404206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204277.404211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204277.404214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204277.404219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204277.404221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204277.404309:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204277.404313:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204277.404316:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204278.428179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204278.428184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204278.428191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204278.428194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204278.428198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204278.428201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204278.428207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204278.428210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204278.428287:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204278.428291:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204278.428294:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204279.452237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204279.452246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204279.452256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204279.452277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204279.452286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204279.452291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204279.452303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204279.452307:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204279.452375:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204279.452380:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204279.452384:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204280.476194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204280.476201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204280.476210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204280.476213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204280.476220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204280.476223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204280.476229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204280.476232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204280.476311:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204280.476316:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204280.476321:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204280.668288:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204280.668303:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204280.668306:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204280.668311:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204280.668316:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204280.668324:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25967 00000400:00000200:0.0:1788204280.668341:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204280.668348:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204280.668352:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204280.668355:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668357:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.668359:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668361:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204280.668365:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204280.668367:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204280.668370:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204280.668374:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204280.668377:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204280.668379:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204280.668382:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668384:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.668386:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668388:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204280.668391:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204280.668394:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668397:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204280.668400:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204280.668409:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204280.668412:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.668414:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204280.668416:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204280.668426:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204280.668430:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204280.668434:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204280.668435:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.668438:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204280.668440:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204280.668445:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204280.668454:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204280.668466:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204280.668473:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543408384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204280.668481:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204280.668485:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204280.668488:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543408384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204280.668506:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204280.668508:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204280.668511:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204280.668512:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204280.668513:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204280.668515:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204280.668517:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25967 00000400:00000200:0.0:1788204280.668519:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204280.668522:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204280.668523:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204280.668525:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668527:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.668528:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668529:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204280.668530:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204280.668531:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204280.668532:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204280.668534:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204280.668536:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204280.668537:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204280.668541:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668543:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.668546:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668548:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204280.668551:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204280.668554:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668557:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204280.668559:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204280.668567:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204280.668573:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.668576:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204280.668582:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204280.668589:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.668594:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204280.668601:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204280.668605:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204280.668610:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204280.668613:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.668616:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204280.668620:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204280.668624:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204280.668630:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204280.668637:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204280.668646:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543408512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204280.668658:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204280.668661:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204280.668665:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543408512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204280.668684:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204280.668690:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204280.668696:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204280.668708:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204280.668711:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204280.668715:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204280.668722:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25967 00000400:00000200:0.0:1788204280.668730:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204280.668738:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204280.668742:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204280.668750:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668760:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.668767:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668772:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204280.668776:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204280.668779:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204280.668781:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204280.668783:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204280.668788:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204280.668792:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204280.668797:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668802:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.668806:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668811:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204280.668815:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204280.668820:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668823:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204280.668827:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204280.668832:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204280.668835:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.668839:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204280.668843:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204280.668847:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.668849:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204280.668856:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204280.668860:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204280.668865:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204280.668867:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.668869:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204280.668871:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204280.668874:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204280.668881:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204280.668888:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204280.668897:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543408640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204280.668909:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204280.668912:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204280.668920:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543408640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204280.668933:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204280.668934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204280.668937:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204280.668942:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00080000:0.0:1788204280.668943:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204280.668946:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204280.668949:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204280.668955:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204280.668962:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25967 00000400:00000200:0.0:1788204280.668968:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204280.668973:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204280.668978:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204280.668984:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668987:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.668992:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.668996:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204280.668999:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204280.669002:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204280.669004:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204280.669007:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204280.669012:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204280.669017:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204280.669024:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.669027:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.669033:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.669038:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204280.669043:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204280.669048:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204280.669053:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204280.669057:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204280.669061:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204280.669065:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.669067:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204280.669070:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204280.669074:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.669078:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204280.669083:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204280.669087:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204280.669146:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204280.669149:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.669154:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204280.669156:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204280.669160:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204280.669167:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204280.669174:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204280.669184:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543408768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204280.669195:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204280.669200:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204280.669208:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543408768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204280.669221:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204280.669227:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204280.669235:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204280.669250:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204280.669252:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204280.669258:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204280.669263:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25967 00000400:00000200:0.0:1788204280.669272:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204280.669278:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204280.669282:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204280.669288:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.669293:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.669299:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.669303:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204280.669308:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204280.669311:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204280.669314:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204280.669318:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204280.669323:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204280.669327:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204280.669335:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.669340:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204280.669345:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204280.669350:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204280.669355:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204280.669360:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204280.669362:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204280.669363:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204280.669366:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204280.669369:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.669373:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204280.669377:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204280.669384:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.669387:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204280.669393:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204280.669398:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204280.669404:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204280.669406:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204280.669409:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204280.669412:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204280.669414:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204280.669421:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204280.669427:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204280.669436:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543408896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204280.669449:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204280.669453:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204280.669461:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543408896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204280.669474:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204280.669480:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25977) 00000800:00000200:3.1:1788204280.669616:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204280.669618:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204280.669829:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204280.669830:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204280.670022:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204280.670024:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204280.670263:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204280.670266:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788204281.500209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204281.500217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204281.500253:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204281.500257:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204281.500265:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500270:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204281.500273:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204281.500282:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204281.500283:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204281.500285:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500286:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204281.500288:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500290:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500292:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204281.500296:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204281.500297:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204281.500298:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500299:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204281.500300:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500302:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204281.500304:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204281.500367:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204281.500373:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204281.500377:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204282.012227:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320640773, at 4320640832 00000100:00000010:2.0:1788204282.012251:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc900. 00010000:00000001:0.0:1788204282.524395:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204282.524398:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204282.524403:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204282.524404:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204282.524406:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204282.524408:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204282.524411:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204282.524412:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204282.524484:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204282.524488:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204282.524491:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204283.548215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204283.548219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204283.548226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204283.548229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204283.548232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204283.548235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204283.548240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204283.548243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204283.548331:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204283.548337:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204283.548341:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204284.572212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204284.572218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204284.572223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204284.572226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204284.572233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204284.572237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204284.572240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204284.572244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204284.572303:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204284.572307:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204284.572311:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204285.596200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204285.596206:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204285.596210:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204285.596215:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204285.596221:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204285.596224:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204285.596229:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204285.596234:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204285.596242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204285.596244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204285.596247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204285.596249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204285.596252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204285.596253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204285.596305:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204285.596308:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204285.596310:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204285.788215:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204285.788223:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204285.788226:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204285.788228:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204285.788232:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204285.788238:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25972 00000400:00000200:0.0:1788204285.788246:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204285.788252:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204285.788254:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204285.788256:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788258:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788260:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788262:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204285.788265:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204285.788267:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204285.788269:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204285.788272:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204285.788273:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204285.788275:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204285.788277:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788278:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788279:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788280:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204285.788282:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204285.788284:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788286:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204285.788290:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204285.788293:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204285.788295:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788297:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204285.788300:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204285.788308:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204285.788312:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204285.788315:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204285.788317:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788320:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204285.788323:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204285.788328:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204285.788334:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204285.788340:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204285.788348:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543409024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204285.788355:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204285.788358:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204285.788362:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543409024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204285.788381:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204285.788383:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204285.788386:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204285.788387:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204285.788388:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204285.788390:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204285.788392:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25972 00000400:00000200:0.0:1788204285.788394:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204285.788397:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204285.788399:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204285.788401:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788404:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788405:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788407:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204285.788410:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204285.788412:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204285.788414:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204285.788416:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204285.788419:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204285.788422:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204285.788427:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788432:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788444:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788449:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204285.788451:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204285.788455:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788458:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204285.788461:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204285.788467:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204285.788472:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788474:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204285.788477:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204285.788479:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788480:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204285.788483:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204285.788486:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204285.788489:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204285.788492:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788495:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204285.788496:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204285.788498:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204285.788502:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204285.788506:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204285.788514:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543409152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204285.788525:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204285.788528:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204285.788536:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543409152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204285.788548:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204285.788553:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204285.788559:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204285.788560:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204285.788563:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204285.788567:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204285.788570:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25972 00000400:00000200:0.0:1788204285.788578:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204285.788582:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204285.788584:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204285.788588:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788590:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788594:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788595:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204285.788597:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204285.788599:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204285.788601:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204285.788604:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204285.788605:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204285.788606:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204285.788611:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788614:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788616:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788618:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204285.788621:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204285.788626:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788628:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204285.788631:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204285.788634:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204285.788638:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788641:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204285.788643:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204285.788648:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788652:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204285.788656:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204285.788659:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204285.788661:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204285.788663:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788667:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204285.788670:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204285.788672:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204285.788676:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204285.788681:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204285.788687:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543409280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204285.788694:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204285.788696:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204285.788701:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543409280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204285.788713:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204285.788715:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204285.788719:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788204285.788719:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00080000:0.0:1788204285.788725:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204285.788727:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204285.788729:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204285.788732:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204285.788737:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25972 00000400:00000200:0.0:1788204285.788742:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204285.788744:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204285.788745:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204285.788747:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788752:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788756:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788758:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204285.788761:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204285.788763:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204285.788765:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204285.788766:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204285.788773:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204285.788776:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204285.788779:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788782:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788786:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788788:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204285.788791:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204285.788794:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788797:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204285.788800:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204285.788803:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204285.788805:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788809:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204285.788811:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204285.788814:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788817:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204285.788821:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204285.788824:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204285.788827:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204285.788829:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788831:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204285.788833:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204285.788835:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204285.788839:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204285.788842:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204285.788848:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543409408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204285.788855:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204285.788858:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204285.788862:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543409408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204285.788871:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204285.788875:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204285.788877:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204285.788878:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204285.788884:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204285.788890:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204285.788894:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25972 00000400:00000200:0.0:1788204285.788900:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204285.788904:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204285.788907:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204285.788910:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788914:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788918:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788922:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204285.788926:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204285.788928:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204285.788930:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204285.788933:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204285.788936:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204285.788938:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204285.788942:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788944:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204285.788948:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788951:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204285.788954:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204285.788958:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204285.788962:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204285.788966:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204285.788969:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204285.788972:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788975:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204285.788977:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204285.788982:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.788984:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204285.788988:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204285.788992:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204285.788995:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204285.788996:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204285.789000:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204285.789002:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204285.789005:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204285.789008:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204285.789012:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204285.789019:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543409536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204285.789032:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204285.789035:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204285.789040:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543409536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204285.789053:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204285.789057:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25982) 00000800:00000200:3.1:1788204285.789223:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204285.789225:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204285.789387:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204285.789389:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204285.789538:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204285.789541:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204285.789740:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204285.789742:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204286.620193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204286.620198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204286.620203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204286.620205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204286.620212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204286.620215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204286.620219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204286.620223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204286.620277:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204286.620282:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204286.620285:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204287.132209:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320645893, at 4320645952 00000100:00000010:2.0:1788204287.132224:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc540. 00010000:00000001:0.0:1788204287.644205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204287.644209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204287.644214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204287.644217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204287.644221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204287.644224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204287.644231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204287.644233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204287.644279:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204287.644283:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204287.644286:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204288.668186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204288.668190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204288.668199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204288.668202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204288.668207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204288.668209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204288.668216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204288.668219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204288.668266:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204288.668269:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204288.668271:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204289.692207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204289.692212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204289.692220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204289.692223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204289.692230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204289.692234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204289.692239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204289.692243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204289.692272:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204289.692277:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204289.692282:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204290.716186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204290.716191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204290.716197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204290.716199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204290.716206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204290.716210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204290.716215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204290.716219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204290.716273:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204290.716277:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204290.716280:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204290.908212:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204290.908224:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204290.908226:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204290.908230:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204290.908234:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204290.908242:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25977 00000400:00000200:2.0:1788204290.908252:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204290.908258:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204290.908261:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204290.908264:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908266:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908268:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908270:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204290.908273:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204290.908274:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204290.908277:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204290.908281:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204290.908282:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204290.908284:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204290.908286:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908287:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908288:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908289:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204290.908291:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204290.908294:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908296:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204290.908298:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204290.908301:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204290.908304:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908307:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204290.908309:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204290.908319:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204290.908323:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204290.908327:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204290.908329:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908332:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204290.908335:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204290.908340:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204290.908346:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204290.908352:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204290.908360:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543409664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204290.908368:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204290.908371:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204290.908375:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543409664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204290.908392:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204290.908395:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204290.908398:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204290.908399:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204290.908400:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204290.908402:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204290.908405:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25977 00000400:00000200:2.0:1788204290.908408:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204290.908411:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204290.908414:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204290.908417:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908420:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908422:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908425:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204290.908427:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204290.908428:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204290.908430:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204290.908432:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204290.908435:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204290.908437:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204290.908441:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908445:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908449:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908452:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204290.908455:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204290.908459:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908464:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204290.908467:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204290.908472:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204290.908475:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908478:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204290.908482:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204290.908487:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908490:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204290.908495:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204290.908499:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204290.908502:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204290.908504:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908508:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204290.908509:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204290.908512:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204290.908515:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204290.908520:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204290.908529:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543409792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204290.908541:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204290.908545:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204290.908555:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543409792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204290.908566:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204290.908569:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204290.908574:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204290.908577:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204290.908580:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204290.908583:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204290.908586:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25977 00000400:00000200:2.0:1788204290.908591:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204290.908596:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204290.908600:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204290.908604:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908607:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908609:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908613:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204290.908616:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204290.908619:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204290.908620:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204290.908623:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204290.908626:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204290.908631:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204290.908633:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908635:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908639:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908641:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204290.908643:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204290.908645:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908647:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204290.908649:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204290.908653:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204290.908657:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908660:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204290.908664:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204290.908668:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908672:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204290.908678:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204290.908683:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204290.908686:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204290.908687:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908689:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204290.908690:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204290.908692:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204290.908697:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204290.908702:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204290.908709:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543409920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204290.908724:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204290.908727:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204290.908733:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543409920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204290.908744:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204290.908749:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204290.908751:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204290.908754:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204290.908756:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204290.908758:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204290.908761:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25977 00000400:00000200:2.0:1788204290.908764:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204290.908769:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204290.908773:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204290.908776:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908781:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908783:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908787:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788204290.908788:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788204290.908789:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204290.908791:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204290.908794:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:3.1:1788204290.908794:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000020:00000040:2.0:1788204290.908796:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204290.908807:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204290.908810:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204290.908813:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908814:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908819:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908823:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204290.908825:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204290.908830:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908832:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204290.908834:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204290.908839:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204290.908844:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908846:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204290.908850:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204290.908853:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908857:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204290.908863:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204290.908868:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204290.908872:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204290.908874:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.908878:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204290.908879:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204290.908882:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204290.908888:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204290.908893:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204290.908901:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543410048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204290.908916:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204290.908921:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204290.908927:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543410048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204290.908935:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204290.908937:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204290.908944:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204290.908945:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204290.908947:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204290.908953:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204290.908959:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25977 00000400:00000200:2.0:1788204290.908968:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204290.908976:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204290.908981:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204290.908988:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908992:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.908995:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.908997:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204290.909000:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204290.909003:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204290.909005:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204290.909009:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204290.909011:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204290.909014:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204290.909020:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.909025:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204290.909029:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204290.909032:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204290.909035:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204290.909040:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204290.909045:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204290.909048:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204290.909052:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204290.909057:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.909075:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204290.909078:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204290.909081:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.909085:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204290.909111:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204290.909114:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204290.909116:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204290.909118:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204290.909119:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204290.909121:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204290.909122:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204290.909126:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204290.909131:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204290.909137:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543410176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204290.909144:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204290.909148:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204290.909154:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543410176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204290.909171:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204290.909176:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25987) 00000800:00000200:3.1:1788204290.909410:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204290.909412:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204290.909600:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204290.909602:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204290.909788:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204290.909790:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204290.909947:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204290.909949:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204291.740185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204291.740191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204291.740206:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204291.740210:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204291.740215:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740220:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204291.740225:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740227:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740229:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204291.740234:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204291.740235:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204291.740237:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740238:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204291.740239:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740241:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740243:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204291.740248:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204291.740255:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204291.740256:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740258:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204291.740259:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740261:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204291.740263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204291.740313:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204291.740316:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204291.740318:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204292.252190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320651013, at 4320651072 00000100:00000010:2.0:1788204292.252206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc120. 00010000:00000001:0.0:1788204292.764181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204292.764186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204292.764194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204292.764198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204292.764205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204292.764209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204292.764213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204292.764214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204292.764273:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204292.764277:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204292.764280:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204293.788181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204293.788185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204293.788193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204293.788195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204293.788201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204293.788204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204293.788208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204293.788211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204293.788254:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204293.788259:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204293.788262:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204294.812166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204294.812172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204294.812178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204294.812179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204294.812182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204294.812184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204294.812188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204294.812189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204294.812296:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204294.812302:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204294.812307:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204295.836241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204295.836248:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204295.836253:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204295.836258:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204295.836265:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204295.836270:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204295.836274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204295.836283:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204295.836290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204295.836292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204295.836296:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204295.836297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204295.836301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204295.836302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204295.836385:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204295.836391:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204295.836395:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204296.028212:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204296.028222:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204296.028225:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204296.028228:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204296.028239:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204296.028246:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 25982 00000400:00000200:2.0:1788204296.028257:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204296.028265:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204296.028268:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204296.028271:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028274:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.028276:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028279:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204296.028283:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204296.028285:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204296.028287:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204296.028291:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204296.028293:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204296.028296:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204296.028299:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028301:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.028303:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028305:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204296.028308:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204296.028311:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028313:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204296.028316:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204296.028320:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204296.028324:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.028326:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204296.028329:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204296.028338:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204296.028342:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204296.028347:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204296.028349:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.028352:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204296.028355:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204296.028360:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204296.028368:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204296.028374:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204296.028384:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543410304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204296.028395:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204296.028400:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204296.028404:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543410304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204296.028427:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204296.028431:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204296.028434:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204296.028436:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204296.028437:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204296.028441:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204296.028445:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25982 00000400:00000200:2.0:1788204296.028450:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204296.028454:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204296.028457:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204296.028460:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028463:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.028465:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028469:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204296.028473:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204296.028476:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204296.028480:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204296.028487:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204296.028493:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204296.028495:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204296.028502:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028507:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.028515:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028525:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204296.028533:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204296.028541:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028546:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204296.028552:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204296.028562:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204296.028567:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.028574:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204296.028580:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204296.028589:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.028594:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204296.028601:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204296.028607:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204296.028611:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204296.028613:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.028617:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204296.028620:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204296.028624:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204296.028631:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204296.028645:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204296.028657:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543410432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204296.028688:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204296.028699:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204296.028712:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543410432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204296.028737:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204296.028741:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204296.028744:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204296.028748:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204296.028752:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204296.028761:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204296.028764:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25982 00000400:00000200:2.0:1788204296.028820:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204296.028825:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204296.028831:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204296.028834:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028840:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.028850:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028861:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788204296.028865:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788204296.028871:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:3.1:1788204296.028872:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000020:00000040:2.0:1788204296.028877:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204296.028883:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204296.028889:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204296.028894:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204296.028902:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204296.028910:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028917:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.028924:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028928:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204296.028936:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204296.028946:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204296.028956:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204296.028968:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204296.028983:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204296.028988:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.028994:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204296.028998:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204296.029006:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.029015:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204296.029020:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204296.029029:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204296.029032:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204296.029034:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.029038:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204296.029042:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204296.029046:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204296.029052:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204296.029057:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204296.029066:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543410560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204296.029074:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204296.029082:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204296.029114:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543410560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204296.029120:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204296.029124:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204296.029126:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204296.029127:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204296.029131:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204296.029134:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204296.029138:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25982 00000400:00000200:2.0:1788204296.029152:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204296.029162:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204296.029169:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204296.029182:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029192:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.029202:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029205:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204296.029207:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204296.029213:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204296.029217:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204296.029220:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204296.029230:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204296.029232:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204296.029241:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029247:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.029252:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029254:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204296.029262:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204296.029277:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029285:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204296.029291:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204296.029295:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204296.029305:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.029314:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204296.029318:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204296.029322:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.029327:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204296.029335:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204296.029338:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204296.029339:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204296.029340:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.029341:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204296.029345:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204296.029348:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204296.029352:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204296.029361:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204296.029373:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543410688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204296.029387:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204296.029392:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204296.029405:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543410688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204296.029444:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204296.029452:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204296.029465:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204296.029471:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204296.029477:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204296.029484:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204296.029493:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 25982 00000400:00000200:2.0:1788204296.029509:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204296.029525:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204296.029532:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204296.029542:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029547:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.029555:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029569:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204296.029578:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204296.029586:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204296.029589:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204296.029596:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204296.029605:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204296.029617:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204296.029620:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029623:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204296.029629:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029632:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204296.029638:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204296.029641:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204296.029642:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204296.029643:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204296.029647:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204296.029651:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.029655:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204296.029659:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204296.029664:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.029671:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204296.029685:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204296.029695:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204296.029700:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204296.029703:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204296.029708:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204296.029712:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204296.029714:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204296.029718:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204296.029724:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204296.029734:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543410816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204296.029746:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204296.029749:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204296.029756:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543410816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204296.029782:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204296.029788:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (25992) 00000800:00000200:3.1:1788204296.029852:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204296.029854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204296.030044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204296.030045:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204296.030256:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204296.030258:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204296.030486:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204296.030488:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1788204296.860242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204296.860250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204296.860264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204296.860273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204296.860281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204296.860285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204296.860299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204296.860302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204296.860371:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204296.860374:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204296.860377:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204297.372192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320656133, at 4320656192 00000100:00000010:2.0:1788204297.372205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcc60. 00010000:00000001:0.0:1788204297.884192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204297.884196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204297.884203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204297.884206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204297.884210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204297.884213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204297.884220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204297.884223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204297.884279:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204297.884283:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204297.884285:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204298.908190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204298.908197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204298.908203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204298.908208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204298.908216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204298.908221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204298.908229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204298.908231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204298.908286:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204298.908288:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204298.908290:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204299.932181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204299.932185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204299.932194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204299.932198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204299.932205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204299.932209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204299.932217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204299.932220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204299.932276:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204299.932280:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204299.932283:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204300.956226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204300.956237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204300.956249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204300.956254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204300.956260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204300.956268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204300.956282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204300.956291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204300.956359:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204300.956362:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204300.956365:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204301.149077:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204301.149084:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204301.149705:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204301.149707:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204301.150007:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204301.150009:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204301.150295:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204301.150297:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204301.150577:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204301.150580:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00010000:00000001:0.0:1788204301.980188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204301.980193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204301.980206:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204301.980211:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204301.980216:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980224:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204301.980229:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980247:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204301.980263:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204301.980267:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204301.980270:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980274:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204301.980276:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980281:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980283:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204301.980289:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204301.980290:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204301.980291:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980292:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204301.980293:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980294:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204301.980296:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204301.980359:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204301.980364:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204301.980367:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204302.492199:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320661254, at 4320661312 00000100:00000010:2.0:1788204302.492215:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc360. 00010000:00000001:0.0:1788204303.004250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204303.004260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204303.004278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204303.004283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204303.004294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204303.004300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204303.004311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204303.004317:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204303.004377:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204303.004381:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204303.004385:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204304.028238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204304.028246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204304.028261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204304.028267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204304.028275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204304.028282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204304.028299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204304.028302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204304.028358:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204304.028362:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204304.028364:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204305.052239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204305.052245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204305.052257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204305.052261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204305.052267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204305.052271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204305.052279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204305.052283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204305.052361:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204305.052366:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204305.052368:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204306.076217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204306.076226:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204306.076231:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204306.076239:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204306.076246:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204306.076251:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204306.076254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204306.076260:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204306.076269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204306.076271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204306.076275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204306.076276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204306.076280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204306.076282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204306.076334:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204306.076338:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204306.076340:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204306.268745:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204306.269175:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204306.269363:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204306.269516:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204306.269655:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204307.100214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204307.100222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204307.100234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204307.100240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204307.100251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204307.100258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204307.100276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204307.100279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204307.100347:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204307.100351:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204307.100354:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204307.612231:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320666373, at 4320666432 00000100:00000010:2.0:1788204307.612252:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fce40. 00010000:00000001:0.0:1788204308.124229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204308.124234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204308.124244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204308.124248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204308.124254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204308.124257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204308.124265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204308.124270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204308.124333:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204308.124338:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204308.124342:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204309.148240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204309.148248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204309.148259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204309.148263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204309.148275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204309.148278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204309.148283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204309.148287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204309.148355:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204309.148360:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204309.148363:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204310.172198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204310.172204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204310.172211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204310.172215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204310.172223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204310.172227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204310.172235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204310.172238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204310.172304:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204310.172310:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204310.172314:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00000020:0.0:1788204310.876212:0:417027:0:(osc_request.c:745:osc_update_next_shrink()) next time 27202 to shrink grant 00000008:00000020:0.0:1788204310.876220:0:417027:0:(osc_request.c:745:osc_update_next_shrink()) next time 27202 to shrink grant 00010000:00000001:0.0:1788204311.196194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204311.196199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204311.196207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204311.196210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204311.196216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204311.196220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204311.196226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204311.196230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204311.196296:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204311.196307:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204311.196310:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204311.388793:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204311.388801:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204311.389539:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204311.389542:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204311.389901:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204311.389903:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204311.390198:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204311.390201:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204311.390477:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204311.390479:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204312.220160:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204312.220164:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204312.220175:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204312.220177:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204312.220181:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220187:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204312.220190:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220192:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220196:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204312.220201:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204312.220202:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204312.220203:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220204:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204312.220205:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220206:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220207:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204312.220217:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204312.220217:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204312.220218:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220219:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204312.220220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220221:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204312.220222:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204312.220266:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204312.220270:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204312.220272:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204312.732175:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320671493, at 4320671552 00000100:00000010:2.0:1788204312.732189:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00010000:00000001:0.0:1788204313.244221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204313.244229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204313.244236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204313.244238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204313.244252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204313.244258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204313.244272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204313.244275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204313.244348:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204313.244352:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204313.244354:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204314.268179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204314.268184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204314.268189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204314.268192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204314.268199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204314.268202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204314.268206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204314.268208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204314.268266:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204314.268271:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204314.268274:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204315.292183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204315.292187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204315.292194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204315.292197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204315.292202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204315.292204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204315.292207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204315.292209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204315.292262:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204315.292265:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204315.292267:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204316.316190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204316.316197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204316.316203:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204316.316209:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204316.316215:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204316.316220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204316.316227:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204316.316232:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204316.316241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204316.316244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204316.316249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204316.316251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204316.316255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204316.316256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204316.316311:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204316.316315:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204316.316318:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204316.508671:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204316.508678:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204316.509257:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204316.509258:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204316.509428:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204316.509429:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204316.509585:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204316.509587:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204316.509820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204316.509821:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00010000:00000001:0.0:1788204317.340186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204317.340190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204317.340197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204317.340200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204317.340205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204317.340208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204317.340213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204317.340216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204317.340317:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204317.340322:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204317.340325:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204317.852171:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320676613, at 4320676672 00000100:00000010:2.0:1788204317.852186:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00010000:00000001:0.0:1788204318.364194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204318.364198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204318.364207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204318.364211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204318.364218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204318.364221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204318.364229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204318.364233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204318.364296:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204318.364299:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204318.364301:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204319.388176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204319.388182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204319.388191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204319.388194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204319.388203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204319.388206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204319.388210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204319.388211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204319.388261:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204319.388264:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204319.388266:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204320.412189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204320.412195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204320.412200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204320.412204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204320.412209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204320.412212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204320.412215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204320.412218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204320.412271:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204320.412274:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204320.412277:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204321.436159:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204321.436164:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204321.436171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204321.436174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204321.436178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204321.436181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204321.436188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204321.436191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204321.436245:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204321.436249:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204321.436252:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204321.628698:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204321.628702:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204321.629336:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204321.629337:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204321.629500:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204321.629502:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204321.629677:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204321.629678:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:3.1:1788204321.630000:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204321.630001:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00010000:00000001:0.0:1788204322.460174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204322.460179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204322.460189:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204322.460191:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204322.460196:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460201:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204322.460203:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460204:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460213:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204322.460219:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204322.460219:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204322.460220:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460221:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204322.460222:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460223:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460225:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204322.460228:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204322.460228:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204322.460229:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460230:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204322.460230:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460231:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204322.460232:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204322.460288:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204322.460291:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204322.460294:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204322.972175:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320681733, at 4320681792 00000100:00000010:2.0:1788204322.972191:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc0c0. 00010000:00000001:0.0:1788204323.484195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204323.484202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204323.484210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204323.484215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204323.484221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204323.484225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204323.484233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204323.484237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204323.484297:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204323.484301:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204323.484303:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204324.508185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204324.508189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204324.508196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204324.508200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204324.508208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204324.508212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204324.508220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204324.508224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204324.508284:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204324.508288:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204324.508290:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204325.532187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204325.532194:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204325.532197:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204325.532208:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204325.532212:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204325.532214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204325.532216:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204325.532217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204325.532222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204325.532224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204325.532226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204325.532227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204325.532229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204325.532230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204325.532295:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204325.532302:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204325.532308:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204326.556203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204326.556209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204326.556221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204326.556225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204326.556232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204326.556235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204326.556245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204326.556249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204326.556307:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204326.556311:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204326.556314:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204326.748231:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204326.748243:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204326.748246:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204326.748250:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204326.748255:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204326.748262:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26013 00000400:00000200:0.0:1788204326.748274:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204326.748282:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204326.748286:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204326.748289:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748293:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.748295:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748298:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204326.748302:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204326.748306:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204326.748310:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204326.748314:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204326.748317:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204326.748319:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204326.748323:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748325:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.748327:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748329:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204326.748333:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204326.748336:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748339:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204326.748342:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204326.748346:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204326.748349:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.748351:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204326.748354:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204326.748364:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204326.748368:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204326.748372:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204326.748373:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.748377:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204326.748380:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204326.748384:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204326.748393:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204326.748400:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204326.748409:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543414144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204326.748421:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204326.748425:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204326.748431:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543414144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204326.748492:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204326.748495:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204326.748499:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204326.748501:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204326.748503:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204326.748506:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204326.748511:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26013 00000400:00000200:0.0:1788204326.748516:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204326.748521:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204326.748527:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204326.748541:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748548:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.748559:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748565:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204326.748571:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204326.748576:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204326.748581:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204326.748586:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204326.748593:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204326.748599:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204326.748606:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748610:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.748617:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748623:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204326.748628:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204326.748633:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748638:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204326.748642:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204326.748651:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204326.748659:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.748664:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204326.748677:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204326.748683:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.748686:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204326.748694:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204326.748700:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204326.748705:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204326.748710:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.748716:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204326.748720:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204326.748726:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204326.748733:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204326.748742:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204326.748754:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543414272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204326.748771:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204326.748777:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204326.748780:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204326.748784:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000040:0.0:1788204326.748787:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543414272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204326.748804:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204326.748811:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204326.748818:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204326.748824:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204326.748829:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204326.748837:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204326.748844:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26013 00000400:00000200:0.0:1788204326.748856:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204326.748866:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204326.748872:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204326.748880:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748886:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.748891:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748897:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204326.748900:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204326.748904:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204326.748907:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204326.748910:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204326.748916:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204326.748922:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204326.748930:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748936:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.748941:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748945:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204326.748949:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204326.748957:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204326.748964:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204326.748967:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204326.748971:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204326.748975:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.748980:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204326.748985:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204326.748990:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.748994:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204326.749001:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204326.749006:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204326.749012:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204326.749015:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.749020:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204326.749023:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204326.749027:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204326.749034:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204326.749041:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204326.749052:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543414400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204326.749065:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204326.749070:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204326.749079:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543414400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204326.749123:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204326.749127:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204326.749131:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204326.749133:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204326.749137:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204326.749145:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204326.749151:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26013 00000400:00000200:0.0:1788204326.749160:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204326.749167:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204326.749173:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204326.749182:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749188:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.749195:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749201:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204326.749208:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204326.749212:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204326.749216:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204326.749220:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204326.749227:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204326.749234:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204326.749243:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749250:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.749257:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749264:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204326.749271:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204326.749280:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749288:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204326.749292:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204326.749296:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204326.749304:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.749310:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204326.749314:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204326.749322:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.749327:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204326.749333:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204326.749338:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204326.749344:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204326.749347:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.749354:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204326.749358:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204326.749362:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204326.749369:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204326.749375:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204326.749388:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543414528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204326.749408:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204326.749412:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204326.749423:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543414528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204326.749460:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204326.749470:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204326.749479:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204326.749483:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204326.749487:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204326.749494:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204326.749502:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26013 00000400:00000200:0.0:1788204326.749513:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204326.749525:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204326.749533:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204326.749542:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749549:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.749557:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788204326.749560:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204326.749564:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788204326.749564:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000020:00000001:0.0:1788204326.749571:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204326.749574:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204326.749578:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204326.749581:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204326.749586:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204326.749591:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204326.749598:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749601:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204326.749606:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749611:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204326.749616:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204326.749622:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204326.749626:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204326.749630:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204326.749633:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204326.749639:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.749642:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204326.749645:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204326.749649:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.749651:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204326.749655:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204326.749659:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204326.749663:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204326.749666:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204326.749672:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204326.749675:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204326.749678:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204326.749684:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204326.749691:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204326.749699:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543414656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204326.749717:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204326.749723:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204326.749734:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543414656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204326.749762:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204326.749768:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26023) 00000800:00000200:3.1:1788204326.749890:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204326.749892:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204326.750207:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204326.750210:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788204326.750498:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204326.750500:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00010000:00000001:0.0:1788204327.580200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204327.580208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204327.580218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204327.580222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204327.580229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204327.580233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204327.580240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204327.580244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204327.580309:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204327.580312:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204327.580315:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204328.092196:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320686853, at 4320686912 00000100:00000010:2.0:1788204328.092209:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc720. 00010000:00000001:0.0:1788204328.604201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204328.604206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204328.604214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204328.604218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204328.604226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204328.604230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204328.604234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204328.604238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204328.604293:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204328.604298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204328.604302:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204329.628181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204329.628185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204329.628191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204329.628193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204329.628196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204329.628199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204329.628203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204329.628205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204329.628258:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204329.628261:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204329.628264:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204330.652191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204330.652196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204330.652206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204330.652210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204330.652216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204330.652219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204330.652227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204330.652231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204330.652290:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204330.652295:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204330.652298:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204331.676174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204331.676181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204331.676196:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204331.676200:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204331.676205:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676210:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204331.676214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676218:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676223:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204331.676229:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204331.676230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204331.676231:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676232:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204331.676233:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676235:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676236:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204331.676239:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204331.676240:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204331.676240:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204331.676242:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676243:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204331.676244:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204331.676276:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204331.676280:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204331.676282:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204331.868215:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204331.868223:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204331.868225:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204331.868227:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204331.868230:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204331.868236:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26018 00000400:00000200:2.0:1788204331.868245:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204331.868251:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204331.868253:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204331.868255:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868258:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868259:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868261:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204331.868264:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204331.868265:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204331.868267:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204331.868269:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204331.868271:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204331.868272:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204331.868274:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868276:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868277:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868278:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204331.868280:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204331.868282:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868284:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204331.868286:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204331.868289:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204331.868291:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868292:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204331.868294:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204331.868300:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204331.868302:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204331.868305:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204331.868306:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868308:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204331.868310:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204331.868313:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204331.868318:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204331.868324:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204331.868332:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543414784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204331.868339:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204331.868342:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204331.868345:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543414784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204331.868374:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204331.868376:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204331.868378:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204331.868379:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204331.868380:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204331.868382:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204331.868384:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26018 00000400:00000200:2.0:1788204331.868387:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204331.868389:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204331.868390:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204331.868392:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868394:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868396:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868398:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204331.868399:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204331.868400:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204331.868401:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204331.868403:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204331.868404:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204331.868405:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204331.868408:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868410:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868413:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868415:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204331.868420:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204331.868424:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868426:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204331.868428:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204331.868432:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204331.868435:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868438:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204331.868442:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204331.868446:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868449:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204331.868452:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204331.868455:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204331.868457:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204331.868460:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868462:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204331.868465:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204331.868468:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204331.868473:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204331.868477:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204331.868485:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543414912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204331.868492:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204331.868496:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204331.868500:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543414912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204331.868511:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204331.868513:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204331.868517:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204331.868520:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204331.868521:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204331.868523:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204331.868526:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26018 00000400:00000200:2.0:1788204331.868537:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204331.868542:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204331.868544:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204331.868549:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868553:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868557:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868559:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204331.868561:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204331.868563:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204331.868566:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204331.868569:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204331.868572:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204331.868577:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204331.868582:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868586:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868589:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868593:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204331.868595:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204331.868599:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868602:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204331.868603:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204331.868605:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204331.868608:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868610:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204331.868613:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204331.868615:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868616:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204331.868618:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204331.868621:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204331.868623:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204331.868626:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868627:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204331.868628:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204331.868630:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204331.868632:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204331.868640:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204331.868645:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543415040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204331.868653:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204331.868656:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204331.868661:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543415040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204331.868672:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204331.868676:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204331.868679:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204331.868680:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204331.868681:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204331.868683:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204331.868687:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26018 00000400:00000200:2.0:1788204331.868699:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204331.868705:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204331.868708:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204331.868710:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868711:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868713:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868717:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204331.868719:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204331.868719:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204331.868720:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204331.868722:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204331.868726:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204331.868730:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204331.868735:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868737:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868741:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788204331.868742:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204331.868746:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788204331.868747:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000020:00000040:2.0:1788204331.868750:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204331.868756:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868758:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204331.868761:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204331.868764:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204331.868767:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868770:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204331.868773:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204331.868776:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868778:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204331.868783:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204331.868785:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204331.868787:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204331.868789:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868791:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204331.868793:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204331.868795:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204331.868799:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204331.868803:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204331.868809:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543415168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204331.868824:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204331.868828:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204331.868836:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543415168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204331.868853:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204331.868856:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204331.868861:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204331.868862:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204331.868863:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204331.868866:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204331.868869:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26018 00000400:00000200:2.0:1788204331.868873:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204331.868877:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204331.868880:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204331.868884:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868889:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868893:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868897:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204331.868900:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204331.868901:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204331.868904:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204331.868906:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204331.868910:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204331.868913:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204331.868917:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868921:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204331.868925:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868928:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204331.868930:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204331.868933:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204331.868937:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204331.868938:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204331.868941:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204331.868943:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868944:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204331.868947:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204331.868951:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868954:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204331.868959:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204331.868962:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204331.868967:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204331.868970:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204331.868976:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204331.868978:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204331.868980:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204331.868986:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204331.868993:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204331.869002:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543415296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204331.869026:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204331.869029:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204331.869036:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543415296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204331.869052:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204331.869056:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26028) 00000800:00000200:3.1:1788204331.869226:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204331.869228:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204331.869424:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204331.869426:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204331.869613:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204331.869615:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204331.869794:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204331.869796:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00010000:00000001:0.0:1788204332.700186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204332.700191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204332.700196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204332.700198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204332.700203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204332.700206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204332.700211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204332.700214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204332.700274:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204332.700278:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204332.700281:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204333.212192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320691973, at 4320692032 00000100:00000010:2.0:1788204333.212210:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc180. 00010000:00000001:0.0:1788204333.724168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204333.724173:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204333.724181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204333.724186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204333.724193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204333.724195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204333.724198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204333.724200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204333.724253:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204333.724256:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204333.724259:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204334.748169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204334.748173:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204334.748178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204334.748181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204334.748183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204334.748186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204334.748190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204334.748192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204334.748244:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204334.748246:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204334.748248:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204335.772189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204335.772196:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204335.772201:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204335.772205:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204335.772212:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204335.772216:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204335.772224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204335.772231:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204335.772240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204335.772243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204335.772247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204335.772248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204335.772253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204335.772255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204335.772313:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204335.772316:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204335.772318:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204336.796172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204336.796176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204336.796184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204336.796187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204336.796191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204336.796195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204336.796200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204336.796204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204336.796253:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204336.796255:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204336.796257:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204336.988180:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204336.988191:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204336.988193:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204336.988195:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204336.988199:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204336.988204:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26023 00000400:00000200:2.0:1788204336.988213:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204336.988218:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204336.988222:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204336.988224:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988227:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988229:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988231:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204336.988234:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204336.988235:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204336.988238:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204336.988241:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204336.988243:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204336.988245:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204336.988247:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988248:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988250:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988251:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204336.988253:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204336.988255:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988257:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204336.988270:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204336.988273:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204336.988275:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988277:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204336.988278:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204336.988285:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204336.988288:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204336.988291:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204336.988292:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988294:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204336.988296:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204336.988299:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204336.988305:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204336.988310:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204336.988319:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543415424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204336.988326:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204336.988330:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204336.988333:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543415424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204336.988348:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204336.988350:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204336.988353:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204336.988353:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204336.988354:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204336.988356:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204336.988358:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26023 00000400:00000200:2.0:1788204336.988360:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204336.988363:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204336.988364:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204336.988367:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988368:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988370:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988372:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204336.988373:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204336.988374:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204336.988375:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204336.988377:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204336.988379:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204336.988381:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204336.988384:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988388:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988389:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988392:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204336.988394:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204336.988397:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988399:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204336.988401:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204336.988406:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204336.988410:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988413:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204336.988418:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204336.988422:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988425:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204336.988427:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204336.988430:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204336.988434:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204336.988436:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988440:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204336.988441:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204336.988444:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204336.988452:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204336.988457:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204336.988466:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543415552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204336.988481:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204336.988486:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204336.988492:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543415552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204336.988505:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204336.988509:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204336.988515:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204336.988519:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204336.988523:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204336.988527:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204336.988534:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26023 00000400:00000200:2.0:1788204336.988540:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204336.988546:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204336.988549:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204336.988553:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988558:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988560:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988563:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204336.988565:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204336.988566:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204336.988567:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204336.988568:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204336.988569:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204336.988570:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204336.988572:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988573:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988574:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988575:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204336.988577:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204336.988581:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988586:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204336.988587:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204336.988590:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204336.988594:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988599:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204336.988603:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204336.988607:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988612:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204336.988617:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204336.988622:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204336.988628:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204336.988631:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988636:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204336.988639:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204336.988648:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204336.988655:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204336.988661:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204336.988670:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543415680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204336.988693:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204336.988697:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204336.988698:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788204336.988702:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543415680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204336.988703:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1788204336.988714:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204336.988719:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204336.988724:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204336.988726:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204336.988727:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204336.988730:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204336.988734:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26023 00000400:00000200:2.0:1788204336.988739:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204336.988744:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204336.988746:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204336.988748:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988750:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988753:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988755:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204336.988759:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204336.988760:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204336.988762:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204336.988766:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204336.988768:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204336.988770:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204336.988774:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988778:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988781:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988784:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204336.988788:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204336.988792:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988794:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204336.988796:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204336.988799:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204336.988802:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988805:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204336.988807:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204336.988811:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988814:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204336.988821:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204336.988826:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204336.988830:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204336.988832:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.988834:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204336.988838:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204336.988841:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204336.988848:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204336.988853:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204336.988862:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543415808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204336.988873:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204336.988877:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204336.988885:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543415808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204336.988923:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204336.988926:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204336.988931:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204336.988934:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204336.988937:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204336.988943:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204336.988949:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26023 00000400:00000200:2.0:1788204336.988958:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204336.988966:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204336.988971:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204336.988979:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988983:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.988989:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.988995:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204336.988999:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204336.989001:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204336.989003:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204336.989005:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204336.989007:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204336.989010:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204336.989016:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.989019:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204336.989022:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204336.989025:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204336.989027:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204336.989031:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204336.989033:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204336.989035:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204336.989039:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204336.989043:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.989046:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204336.989048:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204336.989052:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.989055:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204336.989060:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204336.989063:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204336.989066:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204336.989068:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204336.989071:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204336.989073:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204336.989076:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204336.989080:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204336.989083:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204336.989106:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543415936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204336.989112:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204336.989114:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204336.989118:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543415936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204336.989123:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204336.989126:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26033) 00000800:00000200:3.1:1788204336.989336:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204336.989339:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204336.989599:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204336.989601:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204336.989852:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204336.989855:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204336.990128:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204336.990131:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00010000:00000001:0.0:1788204337.820183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204337.820187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204337.820195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204337.820197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204337.820204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204337.820206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204337.820211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204337.820214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204337.820274:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204337.820277:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204337.820279:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204338.332180:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320697093, at 4320697152 00000100:00000010:2.0:1788204338.332196:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcae0. 00010000:00000001:0.0:1788204338.844212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204338.844217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204338.844241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204338.844245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204338.844255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204338.844261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204338.844266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204338.844274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204338.844342:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204338.844346:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204338.844349:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204339.868193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204339.868199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204339.868205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204339.868206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204339.868231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204339.868234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204339.868247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204339.868254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204339.868332:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204339.868337:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204339.868342:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204340.892181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204340.892186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204340.892203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204340.892205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204340.892212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204340.892215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204340.892219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204340.892222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204340.892277:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204340.892280:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204340.892282:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204341.916204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204341.916209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204341.916225:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204341.916229:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204341.916235:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204341.916246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916254:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204341.916264:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204341.916265:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204341.916267:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916269:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204341.916270:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916272:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916273:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204341.916278:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204341.916280:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204341.916281:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916282:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204341.916284:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916285:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204341.916287:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204341.916355:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204341.916359:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204341.916362:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204342.108799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204342.108805:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204342.109366:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204342.109368:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204342.109603:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204342.109605:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204342.109850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204342.109852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204342.110128:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204342.110131:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1788204342.940175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204342.940180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204342.940189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204342.940193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204342.940200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204342.940204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204342.940211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204342.940215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204342.940268:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204342.940272:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204342.940275:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204343.452192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320702213, at 4320702272 00000100:00000010:2.0:1788204343.452206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00010000:00000001:0.0:1788204343.964196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204343.964200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204343.964206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204343.964210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204343.964212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204343.964216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204343.964219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204343.964222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204343.964292:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204343.964296:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204343.964300:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204344.988186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204344.988189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204344.988197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204344.988200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204344.988205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204344.988208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204344.988215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204344.988219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204344.988270:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204344.988274:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204344.988276:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204346.012210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204346.012218:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204346.012221:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204346.012225:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204346.012230:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204346.012234:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204346.012236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204346.012239:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204346.012246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204346.012249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204346.012252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204346.012253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204346.012256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204346.012258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204346.012307:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204346.012310:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204346.012312:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204347.036203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204347.036209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204347.036218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204347.036227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204347.036233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204347.036236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204347.036241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204347.036245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204347.036300:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204347.036304:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204347.036308:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204347.228810:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204347.228814:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204347.229482:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204347.229484:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204347.229694:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204347.229696:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204347.229872:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204347.229874:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204347.230232:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204347.230233:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788204348.060202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204348.060207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204348.060218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204348.060220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204348.060227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204348.060230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204348.060237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204348.060240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204348.060429:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204348.060431:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204348.060434:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204348.572189:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320707333, at 4320707392 00000100:00000010:2.0:1788204348.572206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647660. 00010000:00000001:0.0:1788204349.084190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204349.084195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204349.084210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204349.084215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204349.084223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204349.084225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204349.084229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204349.084231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204349.084284:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204349.084288:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204349.084291:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204350.108190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204350.108194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204350.108203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204350.108207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204350.108222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204350.108227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204350.108236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204350.108240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204350.108290:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204350.108293:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204350.108295:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204351.132175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204351.132181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204351.132189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204351.132193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204351.132199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204351.132202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204351.132209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204351.132212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204351.132257:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204351.132260:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204351.132263:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204352.156198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204352.156204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204352.156218:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204352.156223:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204352.156228:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204352.156238:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156241:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156243:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204352.156250:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204352.156251:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204352.156252:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156254:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204352.156256:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156257:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156259:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204352.156264:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204352.156266:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204352.156267:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156268:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204352.156270:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156271:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204352.156273:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204352.156316:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204352.156319:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204352.156322:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204352.348695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204352.348700:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204352.349249:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204352.349251:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204352.349438:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204352.349440:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204352.349649:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204352.349651:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204352.349838:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204352.349840:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204353.180194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204353.180201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204353.180210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204353.180213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204353.180217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204353.180221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204353.180227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204353.180230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204353.180262:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204353.180266:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204353.180269:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204353.692184:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320712453, at 4320712512 00000100:00000010:2.0:1788204353.692197:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00010000:00000001:0.0:1788204354.204198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204354.204204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204354.204212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204354.204214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204354.204219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204354.204221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204354.204225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204354.204228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204354.204295:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204354.204299:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204354.204303:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204355.228198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204355.228202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204355.228211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204355.228215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204355.228219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204355.228222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204355.228228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204355.228231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204355.228291:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204355.228295:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204355.228299:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204356.252190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204356.252197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204356.252201:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204356.252206:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204356.252213:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204356.252216:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204356.252222:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204356.252226:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204356.252234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204356.252235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204356.252239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204356.252240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204356.252245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204356.252247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204356.252267:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204356.252270:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204356.252272:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204357.276184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204357.276190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204357.276199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204357.276203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204357.276212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204357.276215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204357.276219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204357.276220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204357.276275:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204357.276278:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204357.276281:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204357.468697:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204357.468702:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204357.469260:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204357.469262:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204357.469476:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204357.469479:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204357.469689:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204357.469691:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204357.469928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204357.469929:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204358.300173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204358.300178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204358.300186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204358.300191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204358.300198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204358.300199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204358.300203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204358.300204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204358.300265:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204358.300269:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204358.300273:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204358.812187:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320717573, at 4320717632 00000100:00000010:2.0:1788204358.812200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc300. 00010000:00000001:0.0:1788204359.324183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204359.324188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204359.324193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204359.324195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204359.324205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204359.324207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204359.324213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204359.324217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204360.348172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204360.348178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204360.348185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204360.348190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204360.348197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204360.348200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204360.348207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204360.348209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204360.348304:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204360.348307:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204360.348310:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204361.372199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204361.372204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204361.372212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204361.372215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204361.372220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204361.372223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204361.372230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204361.372233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204361.372287:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204361.372290:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204361.372292:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204362.396215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204362.396221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204362.396235:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204362.396239:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204362.396244:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396251:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204362.396263:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396270:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396276:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204362.396287:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204362.396290:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204362.396292:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396297:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204362.396299:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396302:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396304:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396308:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204362.396310:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204362.396311:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204362.396313:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396314:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204362.396315:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396317:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204362.396319:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204362.588702:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204362.588706:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204362.589216:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204362.589220:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204362.589618:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204362.589620:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204362.589872:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204362.589874:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204362.590044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204362.590045:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00010000:00000001:0.0:1788204363.420182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204363.420187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204363.420196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204363.420199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204363.420202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204363.420205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204363.420210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204363.420212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204363.420261:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204363.420265:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204363.420268:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204363.932209:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320722693, at 4320722752 00000100:00000010:2.0:1788204363.932224:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf60. 00010000:00000001:0.0:1788204364.444180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204364.444186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204364.444195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204364.444199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204364.444206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204364.444212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204364.444221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204364.444224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204364.444269:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204364.444272:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204364.444274:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204365.468168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204365.468172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204365.468180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204365.468183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204365.468186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204365.468189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204365.468194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204365.468196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204365.468265:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204365.468269:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204365.468272:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204366.492198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204366.492204:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204366.492208:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204366.492212:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204366.492217:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204366.492220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204366.492224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204366.492229:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204366.492235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204366.492237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204366.492242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204366.492245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204366.492249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204366.492251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204366.492316:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204366.492320:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204366.492324:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204367.516192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204367.516197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204367.516206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204367.516210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204367.516217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204367.516221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204367.516227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204367.516231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204367.516285:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204367.516287:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204367.516290:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204367.708213:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204367.708223:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204367.708226:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204367.708229:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204367.708233:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204367.708241:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26054 00000400:00000200:2.0:1788204367.708251:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204367.708258:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204367.708262:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204367.708265:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708268:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.708271:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708273:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204367.708278:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204367.708279:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204367.708282:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204367.708286:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204367.708288:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204367.708291:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204367.708293:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708296:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.708297:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708299:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204367.708302:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204367.708305:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708308:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204367.708311:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204367.708314:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204367.708317:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708319:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204367.708321:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204367.708331:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204367.708335:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204367.708339:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204367.708340:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708343:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204367.708346:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204367.708350:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204367.708359:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204367.708366:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204367.708376:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543419264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204367.708386:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204367.708392:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204367.708396:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543419264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204367.708414:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204367.708416:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204367.708418:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204367.708419:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204367.708421:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204367.708422:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204367.708430:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26054 00000400:00000200:2.0:1788204367.708433:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204367.708435:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204367.708437:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204367.708439:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708441:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.708443:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708444:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204367.708445:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204367.708446:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204367.708448:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204367.708451:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204367.708453:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204367.708456:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204367.708458:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708459:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.708461:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708464:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204367.708465:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204367.708469:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708473:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204367.708476:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204367.708482:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204367.708485:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708488:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204367.708491:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204367.708496:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708500:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204367.708509:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204367.708512:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204367.708516:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204367.708520:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708524:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204367.708528:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204367.708532:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204367.708541:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204367.708547:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204367.708555:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543419392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204367.708571:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204367.708575:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204367.708585:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543419392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204367.708600:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204367.708604:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204367.708607:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204367.708609:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204367.708610:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204367.708612:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204367.708616:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26054 00000400:00000200:2.0:1788204367.708649:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204367.708654:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204367.708656:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204367.708660:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708662:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.708666:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708669:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204367.708671:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204367.708673:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204367.708675:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204367.708677:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000800:00000200:3.1:1788204367.708678:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788204367.708679:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000800:00000200:3.1:1788204367.708681:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000020:00000001:2.0:1788204367.708683:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204367.708689:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708693:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.708702:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708704:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204367.708707:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204367.708713:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708719:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204367.708724:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204367.708753:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204367.708758:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708763:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204367.708766:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204367.708771:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708773:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204367.708776:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204367.708780:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204367.708784:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204367.708786:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708789:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204367.708792:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204367.708794:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204367.708798:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204367.708801:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204367.708808:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543419520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204367.708823:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204367.708826:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204367.708829:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543419520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204367.708841:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204367.708846:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204367.708852:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204367.708854:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204367.708857:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204367.708860:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204367.708868:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26054 00000400:00000200:2.0:1788204367.708876:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204367.708881:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204367.708885:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204367.708890:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708894:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.708898:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708901:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204367.708903:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204367.708906:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204367.708908:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204367.708911:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204367.708914:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204367.708917:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204367.708923:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708925:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.708929:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708931:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204367.708935:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204367.708939:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204367.708943:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204367.708946:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204367.708949:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204367.708950:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708951:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204367.708955:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204367.708959:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708962:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204367.708965:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204367.708969:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204367.708973:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204367.708975:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.708977:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204367.708980:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204367.708982:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204367.708986:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204367.708993:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204367.708999:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543419648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204367.709013:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204367.709016:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204367.709022:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543419648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204367.709031:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204367.709034:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204367.709038:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204367.709039:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204367.709040:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204367.709042:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204367.709046:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26054 00000400:00000200:2.0:1788204367.709052:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204367.709056:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204367.709058:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204367.709062:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.709064:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.709068:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.709072:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204367.709076:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204367.709078:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204367.709081:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204367.709083:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204367.709088:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204367.709119:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204367.709122:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.709125:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204367.709129:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204367.709132:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204367.709135:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204367.709139:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204367.709142:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204367.709143:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204367.709145:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204367.709147:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.709150:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204367.709153:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204367.709157:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.709161:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204367.709165:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204367.709167:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204367.709171:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204367.709173:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204367.709177:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204367.709180:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204367.709182:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204367.709185:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204367.709190:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204367.709196:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543419776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204367.709217:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204367.709221:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204367.709227:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543419776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204367.709237:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204367.709242:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26064) 00000800:00000200:3.1:1788204367.709323:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204367.709325:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204367.709521:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204367.709522:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204367.709676:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204367.709678:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204367.709942:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204367.709944:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00010000:00000001:0.0:1788204368.540183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204368.540188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204368.540196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204368.540199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204368.540206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204368.540210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204368.540213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204368.540217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204368.540282:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204368.540286:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204368.540289:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204369.052217:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320727813, at 4320727872 00000100:00000010:2.0:1788204369.052234:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcd20. 00010000:00000001:0.0:1788204369.564203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204369.564208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204369.564224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204369.564228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204369.564236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204369.564240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204369.564247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204369.564250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204369.564333:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204369.564338:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204369.564342:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204370.588182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204370.588186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204370.588194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204370.588199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204370.588206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204370.588209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204370.588216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204370.588219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204370.588252:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204370.588256:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204370.588260:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204371.612192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204371.612197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204371.612211:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204371.612215:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204371.612228:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204371.612239:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612244:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612248:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204371.612256:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204371.612257:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204371.612258:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204371.612262:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612264:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612267:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204371.612272:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204371.612274:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204371.612275:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612277:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204371.612285:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612287:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204371.612288:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204371.612342:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204371.612345:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204371.612348:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204372.636195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204372.636201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204372.636221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204372.636225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204372.636231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204372.636235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204372.636241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204372.636242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204372.636294:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204372.636301:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204372.636304:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204372.828242:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204372.828261:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204372.828267:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204372.828272:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204372.828277:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204372.828287:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26059 00000400:00000200:2.0:1788204372.828297:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204372.828306:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204372.828310:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204372.828313:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828317:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.828319:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828322:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204372.828326:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204372.828328:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204372.828330:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204372.828334:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204372.828336:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204372.828339:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204372.828342:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828345:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.828347:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828349:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204372.828351:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204372.828355:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828357:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204372.828360:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204372.828363:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204372.828366:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.828368:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204372.828371:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204372.828380:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204372.828384:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204372.828388:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204372.828389:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.828392:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204372.828395:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204372.828399:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204372.828407:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204372.828413:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204372.828424:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543419904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204372.828435:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204372.828440:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204372.828444:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543419904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204372.828466:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204372.828469:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204372.828472:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204372.828474:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204372.828475:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204372.828478:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204372.828481:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26059 00000400:00000200:2.0:1788204372.828485:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204372.828488:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204372.828490:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204372.828493:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828496:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.828498:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828501:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204372.828504:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204372.828505:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204372.828507:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204372.828510:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204372.828512:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204372.828516:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204372.828528:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828536:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.828540:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828543:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204372.828549:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204372.828556:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828566:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204372.828574:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204372.828582:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204372.828588:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.828593:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204372.828597:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204372.828605:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.828612:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204372.828619:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204372.828628:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204372.828635:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204372.828650:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.828655:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204372.828659:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204372.828662:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204372.828666:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204372.828673:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204372.828682:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543420032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204372.828696:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204372.828701:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204372.828729:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543420032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204372.828754:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204372.828763:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204372.828774:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204372.828783:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204372.828787:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204372.828793:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204372.828799:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26059 00000400:00000200:2.0:1788204372.828812:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204372.828819:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204372.828823:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204372.828831:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828837:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.828846:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828855:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204372.828860:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204372.828866:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204372.828870:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204372.828874:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204372.828882:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204372.828889:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204372.828900:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828907:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.828913:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828917:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204372.828922:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204372.828935:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204372.828946:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204372.828951:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204372.828957:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204372.828962:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.828969:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204372.828972:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204372.828979:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.828986:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204372.828992:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:2.0:1788204372.828993:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204372.828998:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204372.829001:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204372.829002:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.829005:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204372.829007:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204372.829009:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204372.829013:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204372.829018:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204372.829030:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543420160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204372.829044:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204372.829048:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204372.829056:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543420160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204372.829080:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204372.829086:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204372.829129:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204372.829132:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204372.829134:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204372.829139:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204372.829147:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26059 00000400:00000200:2.0:1788204372.829159:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204372.829168:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204372.829172:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204372.829181:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829186:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.829193:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829203:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204372.829208:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204372.829212:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204372.829217:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204372.829223:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204372.829236:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204372.829240:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204372.829251:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829258:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.829269:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829277:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204372.829284:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204372.829295:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829299:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204372.829304:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204372.829311:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204372.829322:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.829331:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204372.829338:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204372.829349:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.829359:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204372.829370:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204372.829379:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204372.829389:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204372.829393:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.829400:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204372.829403:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204372.829406:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204372.829413:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204372.829420:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204372.829434:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543420288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204372.829442:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204372.829462:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204372.829467:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204372.829475:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543420288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204372.829509:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204372.829515:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204372.829524:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204372.829527:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204372.829532:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204372.829538:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204372.829547:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26059 00000400:00000200:2.0:1788204372.829566:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204372.829574:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204372.829580:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204372.829587:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829593:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.829601:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829605:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204372.829612:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204372.829617:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204372.829620:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204372.829623:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204372.829626:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204372.829630:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204372.829636:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829655:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204372.829662:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829671:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204372.829680:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204372.829695:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204372.829704:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204372.829711:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204372.829717:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204372.829724:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.829730:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204372.829734:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204372.829742:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.829753:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204372.829762:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204372.829768:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204372.829776:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204372.829779:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204372.829785:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204372.829787:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204372.829795:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204372.829805:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204372.829813:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204372.829827:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543420416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204372.829859:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204372.829868:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204372.829882:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543420416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204372.829918:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204372.829930:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26069) 00000800:00000200:3.1:1788204372.829934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204372.830211:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204372.830355:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204373.660176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204373.660181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204373.660188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204373.660191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204373.660196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204373.660199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204373.660205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204373.660209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204373.660288:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204373.660293:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204373.660297:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204374.172209:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320732933, at 4320732992 00000100:00000010:2.0:1788204374.172223:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcb40. 00010000:00000001:0.0:1788204374.684190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204374.684195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204374.684200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204374.684202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204374.684208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204374.684211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204374.684216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204374.684219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204374.684281:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204374.684284:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204374.684287:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204375.708186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204375.708191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204375.708200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204375.708203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204375.708210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204375.708213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204375.708219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204375.708223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204375.708282:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204375.708285:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204375.708287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788204375.900407:0:256120:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:0.0:1788204375.900457:0:256120:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204375.900476:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204375.900480:0:256120:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:0.0:1788204375.900485:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204375.900490:0:256120:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:0.0:1788204375.900496:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204375.900499:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:0.0:1788204375.900505:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204375.900510:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204375.900517:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734000 00000400:00000010:0.0:1788204375.900520:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734000. 00000100:00000001:0.0:1788204375.900526:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204375.900537:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74700 x1875055543419904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900567:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204375.900572:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543419904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900579:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204375.900581:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204375.900606:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204375.900609:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd400. 00000400:00000200:0.0:1788204375.900612:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204375.900615:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204375.900619:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734330 00000400:00000010:0.0:1788204375.900620:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734330. 00000100:00000001:0.0:1788204375.900624:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204375.900629:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75c00 x1875055543420032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900638:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204375.900641:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75c00 x1875055543420032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900647:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204375.900649:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204375.900655:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204375.900657:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd100. 00000400:00000200:0.0:1788204375.900660:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204375.900663:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204375.900666:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:0.0:1788204375.900668:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:0.0:1788204375.900670:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204375.900674:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75180 x1875055543420160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900682:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204375.900685:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543420160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900691:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204375.900693:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204375.900698:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204375.900701:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:0.0:1788204375.900703:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204375.900707:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204375.900709:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734f68 00000400:00000010:0.0:1788204375.900711:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734f68. 00000100:00000001:0.0:1788204375.900714:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204375.900719:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75880 x1875055543420288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900729:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204375.900733:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543420288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900739:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204375.900741:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204375.900745:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204375.900747:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd900. 00000400:00000200:0.0:1788204375.900750:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204375.900752:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204375.900756:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987343b8 00000400:00000010:0.0:1788204375.900757:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987343b8. 00000100:00000001:0.0:1788204375.900760:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204375.900765:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d76680 x1875055543420416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900772:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204375.900774:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76680 x1875055543420416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204397 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204375.900780:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204375.900782:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1788204376.732227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204376.732240:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204376.732245:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204376.732254:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204376.732263:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204376.732272:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204376.732280:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204376.732284:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204376.732292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204376.732297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204376.732302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204376.732306:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204376.732312:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204376.732314:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204376.732392:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204376.732397:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204376.732401:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204377.756183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204377.756188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204377.756195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204377.756198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204377.756206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204377.756208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204377.756215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204377.756219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204377.756280:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204377.756283:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204377.756285:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204377.948940:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204377.948948:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204377.949715:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204377.951150:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204377.951496:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204377.951498:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204377.951790:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204377.951792:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204377.952130:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204377.952132:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204378.780214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204378.780220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204378.780227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204378.780229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204378.780233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204378.780235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204378.780241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204378.780244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204378.780302:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204378.780305:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204378.780308:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204379.292166:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320738053, at 4320738112 00000100:00000010:2.0:1788204379.292184:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00010000:00000001:0.0:1788204379.804212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204379.804216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204379.804224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204379.804227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204379.804232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204379.804236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204379.804241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204379.804244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204379.804309:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204379.804314:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204379.804317:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204380.828215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204380.828221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204380.828229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204380.828232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204380.828236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204380.828239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204380.828242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204380.828244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204380.828313:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204380.828317:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204380.828321:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204381.852331:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204381.852341:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852349:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204381.852352:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204381.852355:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204381.852358:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852369:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204381.852372:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852373:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852375:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852379:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204381.852380:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204381.852381:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204381.852383:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852384:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204381.852385:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852386:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852387:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852389:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204381.852391:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204381.852391:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204381.852393:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852394:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204381.852395:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852396:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204381.852397:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204381.852462:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204381.852468:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204381.852471:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204382.876233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204382.876239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204382.876248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204382.876251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204382.876257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204382.876260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204382.876266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204382.876269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204382.876341:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204382.876347:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204382.876351:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204383.068275:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204383.068297:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204383.068300:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204383.068305:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204383.068310:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204383.068318:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26069 00000400:00000200:2.0:1788204383.068330:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204383.068338:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204383.068352:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204383.068357:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068360:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.068363:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068366:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204383.068370:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204383.068372:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204383.068375:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204383.068379:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204383.068382:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204383.068384:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204383.068387:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068389:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.068391:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068392:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204383.068395:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204383.068399:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068402:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204383.068405:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204383.068409:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204383.068412:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.068415:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204383.068418:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204383.068429:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204383.068439:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204383.068443:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204383.068445:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.068449:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204383.068452:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204383.068457:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204383.068480:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204383.068496:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204383.068505:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543421184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204383.068517:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204383.068522:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204383.068527:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543421184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204383.068550:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204383.068553:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204383.068557:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204383.068559:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204383.068567:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204383.068570:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204383.068574:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26069 00000400:00000200:2.0:1788204383.068579:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204383.068583:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204383.068585:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204383.068589:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068591:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.068594:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068598:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204383.068602:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204383.068605:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204383.068607:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204383.068611:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204383.068614:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204383.068616:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204383.068625:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068631:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.068636:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068640:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204383.068647:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204383.068655:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068659:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204383.068662:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204383.068671:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204383.068675:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.068686:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204383.068696:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204383.068701:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.068706:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204383.068710:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204383.068713:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204383.068716:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204383.068717:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.068724:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204383.068728:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204383.068731:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204383.068737:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204383.068742:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204383.068753:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543421312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204383.068767:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204383.068771:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204383.068777:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543421312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204383.068793:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204383.068797:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204383.068803:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204383.068807:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204383.068822:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204383.068828:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204383.068835:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26069 00000400:00000200:2.0:1788204383.068844:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204383.068851:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204383.068856:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204383.068861:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068863:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.068866:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068871:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204383.068876:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204383.068879:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204383.068883:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204383.068886:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204383.068891:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204383.068895:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204383.068899:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068904:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.068906:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068909:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204383.068913:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204383.068921:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204383.068925:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204383.068929:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1788204383.068933:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:2.0:1788204383.068934:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204383.068939:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204383.068941:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788204383.068943:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204383.068947:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204383.068952:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.068956:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204383.068961:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204383.068965:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204383.068969:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204383.068972:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.068977:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204383.068980:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204383.068984:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204383.068989:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204383.068995:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204383.069003:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543421440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204383.069014:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204383.069027:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204383.069035:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543421440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204383.069052:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204383.069057:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204383.069061:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204383.069064:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204383.069067:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204383.069072:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204383.069078:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26069 00000400:00000200:2.0:1788204383.069127:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204383.069132:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204383.069135:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204383.069141:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069144:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.069150:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069153:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204383.069158:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204383.069160:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204383.069164:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204383.069167:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204383.069172:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204383.069178:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204383.069184:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069189:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.069195:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069200:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204383.069203:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204383.069209:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069213:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204383.069217:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204383.069223:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204383.069227:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.069231:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204383.069234:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204383.069240:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.069245:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204383.069251:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204383.069255:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204383.069260:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204383.069263:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.069267:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204383.069271:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204383.069274:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204383.069280:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204383.069287:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204383.069296:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543421568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204383.069311:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204383.069315:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204383.069324:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543421568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204383.069339:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204383.069344:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204383.069350:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204383.069355:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204383.069358:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204383.069366:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204383.069372:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26069 00000400:00000200:2.0:1788204383.069381:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204383.069389:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204383.069392:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204383.069399:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069404:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.069408:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069413:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204383.069418:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204383.069421:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204383.069425:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204383.069428:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204383.069434:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204383.069439:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204383.069444:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069448:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204383.069451:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069457:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204383.069472:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204383.069477:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204383.069482:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204383.069484:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204383.069487:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204383.069491:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.069495:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204383.069499:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204383.069504:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.069509:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204383.069515:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204383.069519:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204383.069524:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204383.069526:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204383.069528:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204383.069529:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204383.069531:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204383.069535:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204383.069540:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204383.069548:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543421696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204383.069561:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204383.069565:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204383.069573:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543421696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204383.069590:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204383.069598:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26079) 00000800:00000200:1.1:1788204383.069706:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204383.069708:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204383.069920:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204383.069922:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204383.070187:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204383.070190:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204383.070521:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204383.070524:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788204383.900211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204383.900215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204383.900222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204383.900225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204383.900227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204383.900230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204383.900233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204383.900236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204383.900293:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204383.900297:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204383.900300:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204384.412208:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320743173, at 4320743232 00000100:00000010:2.0:1788204384.412227:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16474e0. 00010000:00000001:0.0:1788204384.924223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204384.924230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204384.924240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204384.924244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204384.924248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204384.924251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204384.924256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204384.924264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204384.924334:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204384.924339:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204384.924343:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204385.948233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204385.948237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204385.948245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204385.948247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204385.948251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204385.948258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204385.948264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204385.948267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204385.948333:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204385.948337:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204385.948340:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204386.972219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204386.972227:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204386.972232:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204386.972236:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204386.972242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204386.972246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204386.972251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204386.972257:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204386.972264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204386.972267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204386.972272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204386.972274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204386.972278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204386.972279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204386.972341:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204386.972345:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204386.972347:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204387.996220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204387.996223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204387.996232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204387.996235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204387.996240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204387.996243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204387.996250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204387.996254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204387.996309:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204387.996312:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204387.996315:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204388.188916:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204388.188923:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204388.189689:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204388.189692:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204388.189896:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204388.189897:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204388.190056:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204388.190058:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204388.190292:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204388.190293:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1788204389.020222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204389.020227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204389.020234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204389.020237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204389.020242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204389.020244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204389.020250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204389.020259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204389.020320:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204389.020324:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204389.020327:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204389.532217:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320748293, at 4320748352 00000100:00000010:2.0:1788204389.532238:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00010000:00000001:0.0:1788204390.044193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204390.044200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204390.044209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204390.044212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204390.044221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204390.044225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204390.044233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204390.044236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204390.044294:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204390.044298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204390.044300:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204391.068233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204391.068241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204391.068252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204391.068255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204391.068262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204391.068265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204391.068274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204391.068279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204391.068362:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204391.068367:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204391.068370:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204392.092215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204392.092222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204392.092249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204392.092253:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204392.092261:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092265:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204392.092267:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092270:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092273:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204392.092286:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204392.092288:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204392.092290:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092292:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204392.092294:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092295:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092297:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204392.092331:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204392.092333:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204392.092334:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092336:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204392.092338:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092339:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204392.092341:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204392.092408:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204392.092412:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204392.092415:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204393.116204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204393.116209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204393.116216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204393.116219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204393.116223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204393.116226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204393.116230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204393.116232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204393.116336:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204393.116341:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204393.116346:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204393.308928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204393.308940:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204393.309652:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204393.309653:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204393.309904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204393.309905:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204393.310078:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204393.310079:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204393.310257:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204393.310259:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1788204394.140224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204394.140231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204394.140250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204394.140255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204394.140263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204394.140267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204394.140285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204394.140290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204394.140355:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204394.140360:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204394.140363:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204394.652257:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320753414, at 4320753472 00000100:00000010:2.0:1788204394.652292:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647720. 00010000:00000001:0.0:1788204395.164216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204395.164232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204395.164243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204395.164247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204395.164289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204395.164292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204395.164297:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204395.164299:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204395.164386:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204395.164392:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204395.164397:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204396.188247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204396.188256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204396.188263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204396.188267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204396.188278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204396.188283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204396.188289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204396.188292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204396.188372:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204396.188376:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204396.188379:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204397.212173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204397.212179:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204397.212183:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204397.212186:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204397.212192:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204397.212196:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204397.212203:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204397.212208:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204397.212230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204397.212234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204397.212238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204397.212241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204397.212245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204397.212247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204397.212301:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204397.212305:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204397.212308:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204398.236185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204398.236191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204398.236223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204398.236227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204398.236233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204398.236236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204398.236239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204398.236241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204398.236297:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204398.236302:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204398.236305:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204398.428751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204398.428754:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204398.429288:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204398.429290:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204398.429469:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204398.429470:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204398.429661:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204398.429664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204398.429877:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204398.429879:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00010000:00000001:0.0:1788204399.260175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204399.260192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204399.260198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204399.260202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204399.260205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204399.260209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204399.260212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204399.260215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204399.260266:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204399.260271:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204399.260274:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204399.772193:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320758533, at 4320758592 00000100:00000010:2.0:1788204399.772217:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc480. 00010000:00000001:0.0:1788204400.284205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204400.284209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204400.284216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204400.284219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204400.284226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204400.284229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204400.284232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204400.284236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204400.284295:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204400.284299:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204400.284303:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204401.308194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204401.308199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204401.308210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204401.308214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204401.308223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204401.308225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204401.308231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204401.308234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204401.308283:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204401.308286:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204401.308288:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204402.332192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204402.332196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204402.332208:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204402.332210:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204402.332212:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204402.332219:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332222:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332227:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204402.332238:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204402.332240:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204402.332243:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332245:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204402.332246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332247:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204402.332253:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204402.332254:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204402.332256:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332258:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204402.332259:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332261:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204402.332263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204402.332313:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204402.332316:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204402.332319:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204403.356200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204403.356203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204403.356209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204403.356213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204403.356216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204403.356219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204403.356223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204403.356227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204403.356284:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204403.356287:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204403.356290:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204403.548201:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204403.548210:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204403.548212:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204403.548214:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204403.548217:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204403.548223:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26089 00000400:00000200:0.0:1788204403.548231:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204403.548236:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204403.548239:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204403.548241:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548243:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548245:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548246:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204403.548249:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204403.548250:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204403.548253:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204403.548255:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204403.548257:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204403.548258:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204403.548260:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548261:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548262:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548264:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204403.548265:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204403.548267:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548269:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204403.548271:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204403.548273:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204403.548275:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548276:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204403.548278:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204403.548286:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204403.548288:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204403.548291:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204403.548293:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548295:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204403.548298:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204403.548301:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204403.548309:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204403.548316:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204403.548335:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543423744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204403.548345:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204403.548351:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204403.548355:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543423744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204403.548379:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204403.548382:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204403.548385:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204403.548386:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204403.548387:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204403.548389:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204403.548391:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26089 00000400:00000200:0.0:1788204403.548393:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204403.548395:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204403.548397:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204403.548400:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548401:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548402:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548404:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204403.548405:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204403.548407:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204403.548408:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204403.548409:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204403.548410:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204403.548412:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204403.548414:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548417:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548419:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548421:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204403.548424:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204403.548426:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548429:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204403.548431:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204403.548438:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204403.548443:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548448:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204403.548453:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204403.548458:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548462:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204403.548466:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204403.548471:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204403.548476:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204403.548479:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548482:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204403.548485:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204403.548489:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204403.548494:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788204403.548497:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204403.548504:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543423872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204403.548510:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204403.548514:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204403.548518:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543423872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204403.548528:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204403.548532:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204403.548537:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204403.548539:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204403.548540:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204403.548541:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204403.548543:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26089 00000400:00000200:0.0:1788204403.548550:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204403.548552:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204403.548554:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204403.548558:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548561:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548565:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548567:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204403.548571:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204403.548573:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204403.548575:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204403.548577:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204403.548580:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204403.548583:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204403.548590:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548595:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548597:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548600:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204403.548604:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204403.548610:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548615:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204403.548617:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204403.548620:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204403.548622:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548624:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204403.548626:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204403.548629:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548632:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204403.548636:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204403.548637:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204403.548641:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204403.548642:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548645:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204403.548647:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204403.548648:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204403.548650:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788204403.548655:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204403.548659:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543424000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204403.548670:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204403.548673:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204403.548680:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543424000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204403.548699:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204403.548706:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204403.548711:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204403.548713:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204403.548715:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204403.548719:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204403.548724:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26089 00000400:00000200:0.0:1788204403.548730:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204403.548736:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204403.548738:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204403.548743:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548746:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548747:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548749:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204403.548751:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204403.548754:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204403.548756:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204403.548758:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204403.548759:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204403.548763:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204403.548767:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548770:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548771:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548774:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204403.548778:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204403.548782:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548784:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204403.548784:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:0.0:1788204403.548787:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.1:1788204403.548791:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 02000000:00000001:0.0:1788204403.548792:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204403.548796:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548798:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204403.548800:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204403.548803:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548808:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204403.548811:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204403.548814:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204403.548816:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204403.548817:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548820:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204403.548822:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204403.548824:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204403.548826:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788204403.548828:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204403.548835:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543424128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204403.548851:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204403.548854:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204403.548860:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543424128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204403.548887:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204403.548890:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204403.548893:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204403.548895:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204403.548897:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204403.548901:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204403.548904:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26089 00000400:00000200:0.0:1788204403.548911:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204403.548917:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204403.548920:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204403.548924:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548928:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548930:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548932:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204403.548934:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204403.548936:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204403.548939:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204403.548940:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204403.548944:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204403.548948:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204403.548951:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548954:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204403.548955:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548956:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204403.548958:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204403.548963:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204403.548965:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204403.548967:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204403.548969:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204403.548972:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548973:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204403.548974:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204403.548977:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.548978:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204403.548989:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204403.548993:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204403.548996:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204403.548998:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204403.549001:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204403.549003:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204403.549006:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204403.549009:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204403.549013:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204403.549018:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543424256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204403.549040:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204403.549042:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204403.549044:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543424256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204403.549053:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204403.549056:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26100) 00000800:00000200:3.1:1788204403.549385:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204403.549387:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204403.549640:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204403.549643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204403.549919:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204403.549921:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204403.550226:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204403.550228:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00010000:00000001:0.0:1788204404.380185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204404.380189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204404.380196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204404.380199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204404.380202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204404.380205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204404.380209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204404.380211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204404.380255:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204404.380258:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204404.380260:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204404.892190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320763653, at 4320763712 00000100:00000010:2.0:1788204404.892204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc600. 00010000:00000001:0.0:1788204405.404172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204405.404178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204405.404187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204405.404190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204405.404197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204405.404200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204405.404207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204405.404209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204405.404269:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204405.404273:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204405.404276:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204406.428181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204406.428187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204406.428195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204406.428199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204406.428207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204406.428211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204406.428217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204406.428220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204406.428274:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204406.428277:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204406.428281:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204407.452184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204407.452191:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204407.452196:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204407.452202:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204407.452210:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204407.452214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204407.452219:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204407.452224:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204407.452229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204407.452231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204407.452235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204407.452237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204407.452240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204407.452241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204407.452292:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204407.452296:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204407.452298:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204408.476184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204408.476189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204408.476197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204408.476207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204408.476211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204408.476213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204408.476217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204408.476220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204408.476268:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204408.476272:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204408.476275:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204408.668161:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204408.668169:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204408.668171:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204408.668174:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204408.668178:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204408.668183:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26095 00000400:00000200:0.0:1788204408.668192:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204408.668197:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204408.668200:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204408.668202:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668205:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.668207:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668208:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204408.668212:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204408.668213:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204408.668215:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204408.668218:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204408.668219:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204408.668221:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204408.668223:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668224:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.668225:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668227:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204408.668228:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204408.668230:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668232:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204408.668233:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204408.668236:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204408.668238:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668239:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204408.668241:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204408.668247:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204408.668250:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204408.668255:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204408.668256:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668259:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204408.668262:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204408.668266:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204408.668273:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204408.668280:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204408.668289:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543424384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204408.668300:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204408.668311:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204408.668314:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543424384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204408.668334:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204408.668336:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204408.668339:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204408.668340:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204408.668341:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204408.668343:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204408.668345:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26095 00000400:00000200:0.0:1788204408.668347:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204408.668349:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204408.668351:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204408.668356:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668358:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.668360:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668361:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204408.668363:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204408.668364:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204408.668365:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204408.668367:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204408.668369:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204408.668371:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204408.668373:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668375:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.668377:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668378:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204408.668379:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204408.668381:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668383:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204408.668384:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204408.668388:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204408.668392:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668401:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204408.668405:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204408.668411:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668412:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204408.668415:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204408.668422:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204408.668431:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204408.668434:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668442:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204408.668446:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204408.668449:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204408.668459:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204408.668464:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204408.668473:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543424512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204408.668498:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204408.668502:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204408.668510:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543424512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204408.668530:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204408.668533:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204408.668535:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204408.668536:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204408.668537:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204408.668539:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204408.668542:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26095 00000400:00000200:0.0:1788204408.668548:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204408.668549:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204408.668552:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204408.668555:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668556:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.668563:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668572:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204408.668576:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204408.668579:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204408.668581:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204408.668582:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204408.668583:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204408.668586:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204408.668592:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668594:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.668596:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668602:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204408.668610:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204408.668620:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668627:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204408.668637:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204408.668642:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204408.668647:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668651:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204408.668656:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204408.668662:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668664:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204408.668669:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204408.668676:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204408.668678:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204408.668680:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668684:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204408.668686:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204408.668691:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204408.668696:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204408.668702:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204408.668708:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543424640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204408.668730:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204408.668733:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204408.668740:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543424640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204408.668764:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204408.668771:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204408.668783:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204408.668791:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204408.668794:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204408.668800:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204408.668812:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26095 00000400:00000200:0.0:1788204408.668829:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204408.668833:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204408.668839:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204408.668846:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668853:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.668857:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668865:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204408.668874:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:3.1:1788204408.668874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204408.668886:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000020:00000040:0.0:1788204408.668887:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204408.668893:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204408.668895:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204408.668905:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204408.668916:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204408.668920:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668926:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.668934:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668939:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204408.668945:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204408.668959:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204408.668966:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204408.668973:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204408.668980:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204408.668987:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.668996:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204408.669005:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204408.669009:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.669014:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204408.669032:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204408.669039:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204408.669047:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204408.669056:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.669061:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204408.669064:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204408.669068:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204408.669081:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204408.669088:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204408.669129:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543424768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204408.669134:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204408.669138:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204408.669147:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543424768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204408.669162:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204408.669166:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204408.669169:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204408.669171:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204408.669174:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204408.669178:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204408.669187:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26095 00000400:00000200:0.0:1788204408.669194:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204408.669201:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204408.669208:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204408.669219:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.669228:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.669237:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.669242:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204408.669247:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204408.669248:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204408.669250:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204408.669251:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204408.669253:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204408.669258:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204408.669263:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.669266:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204408.669274:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204408.669280:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204408.669287:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204408.669295:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204408.669301:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204408.669303:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204408.669310:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204408.669319:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.669327:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204408.669333:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204408.669338:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.669341:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204408.669353:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204408.669363:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204408.669372:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204408.669376:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204408.669386:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204408.669395:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204408.669401:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204408.669411:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204408.669418:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204408.669436:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543424896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204408.669502:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204408.669506:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204408.669514:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543424896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204408.669526:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204408.669543:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26105) 00000800:00000200:3.1:1788204408.669739:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204408.669740:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204408.669913:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204408.669915:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204408.670119:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204408.670122:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204408.670354:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204408.670356:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1788204409.500181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204409.500187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204409.500195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204409.500200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204409.500207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204409.500211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204409.500218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204409.500221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204409.500320:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204409.500323:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204409.500325:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204410.012147:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320768774, at 4320768832 00000100:00000010:2.0:1788204410.012161:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc000. 00010000:00000001:0.0:1788204410.524195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204410.524199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204410.524211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204410.524214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204410.524218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204410.524221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204410.524228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204410.524231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204410.524276:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204410.524279:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204410.524282:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204411.548170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204411.548175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204411.548189:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204411.548193:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204411.548198:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548203:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204411.548206:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548209:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548211:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204411.548216:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204411.548218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204411.548219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548221:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204411.548223:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548227:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204411.548232:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204411.548233:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204411.548235:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548236:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204411.548238:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548239:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204411.548242:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204411.548330:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204411.548334:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204411.548337:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204412.572165:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204412.572170:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204412.572177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204412.572180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204412.572188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204412.572191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204412.572195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204412.572197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204412.572237:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204412.572240:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204412.572242:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204413.596191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204413.596197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204413.596206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204413.596209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204413.596215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204413.596218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204413.596223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204413.596227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204413.596286:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204413.596290:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204413.596294:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204413.788747:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204413.788756:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204413.789266:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204413.789267:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204413.789429:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204413.789430:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204413.789678:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204413.789680:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204413.789839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204413.789841:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788204414.620183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204414.620188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204414.620193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204414.620196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204414.620203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204414.620207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204414.620212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204414.620215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204414.620272:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204414.620276:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204414.620279:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204415.132190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320773893, at 4320773952 00000100:00000010:2.0:1788204415.132202:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc8a0. 00010000:00000001:0.0:1788204415.644187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204415.644192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204415.644197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204415.644200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204415.644207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204415.644210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204415.644215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204415.644218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204415.644265:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204415.644268:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204415.644270:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204416.668180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204416.668189:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204416.668193:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204416.668198:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204416.668205:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204416.668210:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204416.668215:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204416.668218:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204416.668222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204416.668225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204416.668231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204416.668234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204416.668239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204416.668240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204416.668296:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204416.668298:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204416.668300:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204417.692187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204417.692193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204417.692199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204417.692207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204417.692214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204417.692217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204417.692223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204417.692227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204417.692289:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204417.692292:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204417.692295:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204418.716236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204418.716242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204418.716257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204418.716265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204418.716277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204418.716284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204418.716298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204418.716304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204418.716353:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204418.716359:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204418.716364:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204418.908208:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204418.908223:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204418.908226:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204418.908229:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204418.908232:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204418.908238:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26105 00000400:00000200:2.0:1788204418.908246:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204418.908252:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204418.908254:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204418.908256:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908259:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.908260:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908262:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204418.908264:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204418.908266:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204418.908268:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204418.908271:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204418.908272:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204418.908274:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204418.908276:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908277:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.908278:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908279:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204418.908281:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204418.908283:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908285:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204418.908288:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204418.908291:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204418.908293:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908295:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204418.908298:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204418.908306:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204418.908310:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204418.908313:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204418.908315:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908318:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204418.908320:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204418.908324:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204418.908331:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204418.908338:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204418.908348:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543425664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204418.908355:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204418.908358:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204418.908361:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543425664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204418.908380:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204418.908382:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204418.908385:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204418.908385:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204418.908387:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204418.908388:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204418.908390:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26105 00000400:00000200:2.0:1788204418.908394:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204418.908397:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204418.908400:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204418.908403:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908405:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.908407:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908409:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204418.908412:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204418.908414:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204418.908417:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204418.908420:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204418.908424:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204418.908429:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204418.908434:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908436:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.908439:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908441:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204418.908444:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204418.908449:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908453:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204418.908456:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204418.908463:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204418.908468:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908471:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204418.908478:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204418.908483:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908486:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204418.908491:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204418.908495:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204418.908499:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204418.908501:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908504:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204418.908507:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204418.908510:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204418.908515:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204418.908528:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204418.908537:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543425792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204418.908546:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204418.908548:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204418.908551:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543425792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204418.908563:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204418.908566:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204418.908570:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204418.908573:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204418.908574:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204418.908577:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204418.908581:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26105 00000400:00000200:2.0:1788204418.908588:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204418.908592:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204418.908596:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204418.908600:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908603:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.908606:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908608:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204418.908611:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204418.908612:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204418.908613:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204418.908616:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204418.908619:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204418.908621:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204418.908624:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908626:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.908627:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908631:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204418.908633:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204418.908637:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908639:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204418.908640:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204418.908642:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204418.908645:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908649:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204418.908651:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204418.908652:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908653:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204418.908657:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204418.908661:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204418.908663:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204418.908665:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908669:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204418.908672:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204418.908676:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204418.908681:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204418.908687:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204418.908695:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543425920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204418.908704:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204418.908707:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204418.908714:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543425920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204418.908730:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204418.908735:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204418.908742:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204418.908744:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204418.908746:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204418.908751:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204418.908754:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26105 00000400:00000200:2.0:1788204418.908761:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204418.908766:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204418.908769:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204418.908774:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908778:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1788204418.908778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788204418.908781:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908784:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788204418.908784:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000020:00000001:2.0:1788204418.908785:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204418.908787:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204418.908790:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204418.908791:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204418.908794:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204418.908798:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204418.908801:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908803:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.908805:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908806:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204418.908808:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204418.908812:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908815:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204418.908818:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204418.908821:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204418.908824:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908825:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204418.908827:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204418.908831:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908833:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204418.908850:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204418.908854:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204418.908860:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204418.908862:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.908866:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204418.908869:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204418.908872:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204418.908879:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204418.908886:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204418.908895:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543426048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204418.908911:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204418.908914:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204418.908922:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543426048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204418.908935:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204418.908940:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204418.908944:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204418.908948:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204418.908950:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204418.908953:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204418.908959:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26105 00000400:00000200:2.0:1788204418.908967:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204418.908975:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204418.908978:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204418.908985:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.908991:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.908996:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.909001:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204418.909003:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204418.909007:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204418.909011:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204418.909013:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204418.909017:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204418.909022:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204418.909028:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.909032:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204418.909038:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204418.909043:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204418.909048:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204418.909054:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204418.909062:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204418.909066:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204418.909070:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204418.909075:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.909079:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204418.909082:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204418.909087:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.909116:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204418.909119:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204418.909121:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204418.909124:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204418.909127:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204418.909130:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204418.909133:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204418.909135:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204418.909140:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204418.909144:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204418.909152:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543426176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204418.909163:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204418.909166:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204418.909170:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543426176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204418.909179:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204418.909186:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26115) 00000800:00000200:3.1:1788204418.909402:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204418.909404:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204418.909688:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204418.909691:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204418.909977:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204418.909980:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204418.910254:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204418.910256:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204419.740241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204419.740249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204419.740257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204419.740259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204419.740263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204419.740265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204419.740269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204419.740271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204419.740344:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204419.740348:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204419.740352:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204420.252159:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320779013, at 4320779072 00000100:00000010:2.0:1788204420.252171:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc780. 00010000:00000001:0.0:1788204420.764234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204420.764245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204420.764264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204420.764268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204420.764273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204420.764282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204420.764287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204420.764289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204420.764377:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204420.764382:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204420.764385:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204421.788207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204421.788211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204421.788221:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204421.788224:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204421.788229:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204421.788237:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788242:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788245:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204421.788253:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204421.788255:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204421.788258:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788260:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204421.788262:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788264:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788266:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204421.788271:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204421.788273:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204421.788275:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788277:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204421.788279:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788281:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204421.788283:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204421.788304:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204421.788307:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204421.788310:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204422.812180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204422.812184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204422.812191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204422.812195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204422.812198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204422.812200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204422.812204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204422.812207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204422.812256:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204422.812258:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204422.812261:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204423.836192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204423.836196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204423.836206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204423.836210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204423.836218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204423.836222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204423.836227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204423.836230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204423.836279:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204423.836282:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204423.836285:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204424.028771:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204424.028773:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204424.029507:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204424.029508:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204424.029743:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204424.029744:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204424.029943:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204424.029946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204424.030257:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204424.030259:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204424.860178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204424.860183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204424.860192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204424.860195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204424.860200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204424.860202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204424.860205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204424.860207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204424.860226:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204424.860229:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204424.860232:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204425.372188:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320784133, at 4320784192 00000100:00000010:2.0:1788204425.372204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc060. 00010000:00000001:0.0:1788204425.884167:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204425.884174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204425.884180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204425.884183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204425.884188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204425.884191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204425.884196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204425.884199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204425.884224:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204425.884228:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204425.884232:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204426.908197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204426.908210:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204426.908214:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204426.908220:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204426.908223:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204426.908226:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204426.908231:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204426.908236:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204426.908241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204426.908245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204426.908249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204426.908251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204426.908255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204426.908257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204426.908336:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204426.908341:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204426.908344:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204427.932200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204427.932207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204427.932216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204427.932220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204427.932226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204427.932230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204427.932235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204427.932238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204427.932277:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204427.932281:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204427.932285:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204428.956202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204428.956209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204428.956218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204428.956224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204428.956240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204428.956244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204428.956249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204428.956252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204428.956331:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204428.956337:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204428.956341:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204429.148243:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204429.148255:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204429.148258:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204429.148262:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204429.148267:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204429.148276:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26115 00000400:00000200:2.0:1788204429.148288:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204429.148296:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204429.148299:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204429.148302:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148306:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.148309:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148312:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204429.148316:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204429.148318:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204429.148321:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204429.148325:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204429.148327:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204429.148330:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204429.148333:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148334:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.148336:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148338:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204429.148341:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204429.148344:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148347:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204429.148350:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204429.148354:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204429.148357:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.148360:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204429.148363:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204429.148376:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204429.148380:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204429.148384:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204429.148386:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.148390:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204429.148394:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204429.148399:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204429.148408:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204429.148416:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204429.148424:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543426944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204429.148434:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204429.148440:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204429.148445:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543426944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204429.148477:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204429.148481:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204429.148484:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204429.148486:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204429.148487:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204429.148490:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204429.148493:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26115 00000400:00000200:2.0:1788204429.148497:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204429.148500:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204429.148503:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204429.148506:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148509:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.148512:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148514:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204429.148517:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204429.148520:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204429.148523:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204429.148527:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204429.148532:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204429.148535:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204429.148540:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148542:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.148546:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148550:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204429.148553:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204429.148559:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148567:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204429.148571:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204429.148580:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204429.148586:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.148590:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204429.148597:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204429.148603:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.148608:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204429.148614:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204429.148619:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204429.148624:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204429.148627:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.148631:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204429.148634:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204429.148638:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204429.148645:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204429.148651:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204429.148659:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543427072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204429.148710:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204429.148715:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204429.148723:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543427072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204429.148735:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204429.148740:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204429.148747:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204429.148752:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204429.148755:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204429.148761:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204429.148767:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26115 00000400:00000200:2.0:1788204429.148800:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204429.148807:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204429.148812:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204429.148820:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148824:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.148830:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788204429.148833:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204429.148835:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788204429.148839:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000020:00000001:2.0:1788204429.148840:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204429.148843:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204429.148847:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204429.148851:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204429.148855:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204429.148861:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204429.148869:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148873:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.148878:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148882:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204429.148887:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204429.148895:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204429.148900:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204429.148904:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204429.148909:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204429.148914:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.148919:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204429.148924:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204429.148929:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.148931:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204429.148939:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204429.148944:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204429.148949:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204429.148951:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.148954:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204429.148958:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204429.148960:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204429.148966:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204429.148972:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204429.148983:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543427200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204429.148996:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204429.149000:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204429.149008:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543427200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204429.149023:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204429.149029:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204429.149034:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204429.149038:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204429.149040:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204429.149046:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204429.149053:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26115 00000400:00000200:2.0:1788204429.149061:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204429.149067:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204429.149073:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204429.149080:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149085:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.149135:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149138:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204429.149143:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204429.149146:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204429.149151:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204429.149153:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204429.149158:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204429.149162:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204429.149172:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149179:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.149186:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149190:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204429.149196:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204429.149204:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149209:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204429.149214:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204429.149220:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204429.149225:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.149230:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204429.149234:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204429.149240:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.149245:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204429.149251:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204429.149256:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204429.149263:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204429.149266:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.149271:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204429.149275:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204429.149279:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204429.149286:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204429.149292:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204429.149301:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543427328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204429.149317:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204429.149319:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204429.149325:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543427328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204429.149344:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204429.149350:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204429.149357:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204429.149362:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204429.149365:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204429.149370:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204429.149377:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26115 00000400:00000200:2.0:1788204429.149386:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204429.149393:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204429.149399:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204429.149407:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149413:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.149419:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149423:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204429.149427:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204429.149430:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204429.149433:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204429.149437:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204429.149441:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204429.149447:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204429.149455:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149459:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204429.149465:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149469:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204429.149475:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204429.149483:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204429.149490:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204429.149495:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204429.149499:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204429.149504:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.149509:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204429.149514:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204429.149520:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.149526:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204429.149532:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:2.0:1788204429.149533:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000800:00000200:3.1:1788204429.149535:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000020:00000040:2.0:1788204429.149538:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204429.149545:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204429.149548:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204429.149552:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204429.149555:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204429.149558:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204429.149564:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204429.149571:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204429.149580:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543427456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204429.149606:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204429.149611:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204429.149620:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543427456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204429.149635:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204429.149643:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26125) 00000800:00000200:3.1:1788204429.149840:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204429.149842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204429.150108:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204429.150109:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204429.150348:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204429.150351:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1788204429.980237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204429.980244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204429.980256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204429.980258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204429.980263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204429.980265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204429.980269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204429.980272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204429.980332:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204429.980336:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204429.980340:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204430.492161:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320789253, at 4320789312 00000100:00000010:2.0:1788204430.492173:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc1e0. 00010000:00000001:0.0:1788204431.004164:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204431.004169:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204431.004174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204431.004178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204431.004184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204431.004186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204431.004193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204431.004194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204431.004211:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204431.004212:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204431.004214:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204432.028198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204432.028203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204432.028212:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204432.028215:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204432.028219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028226:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204432.028229:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028238:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204432.028246:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204432.028248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204432.028251:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028253:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204432.028255:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028257:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028259:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204432.028264:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204432.028265:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204432.028265:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028266:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204432.028267:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028269:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204432.028271:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204432.028316:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204432.028319:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204432.028321:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204433.052215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204433.052219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204433.052226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204433.052229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204433.052233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204433.052236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204433.052241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204433.052243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204433.052322:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204433.052330:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204433.052334:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204434.076189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204434.076194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204434.076203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204434.076207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204434.076215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204434.076220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204434.076227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204434.076230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204434.076257:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204434.076261:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204434.076264:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204434.268794:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204434.268799:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204434.269373:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204434.269375:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204434.269673:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204434.269676:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204434.269958:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204434.269961:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204434.270259:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204434.270261:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00010000:00000001:0.0:1788204435.100170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204435.100173:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204435.100180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204435.100183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204435.100190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204435.100193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204435.100200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204435.100203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204435.100246:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204435.100249:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204435.100251:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204435.612304:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320794373, at 4320794432 00000100:00000010:2.0:1788204435.612310:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00010000:00000001:0.0:1788204436.124188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204436.124194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204436.124202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204436.124205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204436.124211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204436.124213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204436.124217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204436.124218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204436.124259:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204436.124262:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204436.124264:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204437.148187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204437.148192:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204437.148196:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204437.148200:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204437.148205:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204437.148209:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204437.148212:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204437.148217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204437.148221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204437.148223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204437.148228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204437.148230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204437.148236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204437.148239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204437.148255:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204437.148257:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204437.148259:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204438.172184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204438.172188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204438.172193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204438.172195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204438.172208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204438.172210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204438.172214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204438.172216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204438.172272:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204438.172277:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204438.172280:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204439.196190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204439.196196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204439.196205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204439.196210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204439.196218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204439.196222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204439.196227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204439.196228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204439.196286:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204439.196290:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204439.196293:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204439.388765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204439.389205:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204439.389358:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204439.389515:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204439.389635:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204440.220198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204440.220203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204440.220211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204440.220214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204440.220219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204440.220222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204440.220226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204440.220229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204440.220268:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204440.220272:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204440.220276:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204440.732188:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320799493, at 4320799552 00000100:00000010:2.0:1788204440.732205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00010000:00000001:0.0:1788204441.244199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204441.244204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204441.244209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204441.244210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204441.244213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204441.244216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204441.244220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204441.244221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204441.244279:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204441.244282:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204441.244285:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204442.268183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204442.268191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204442.268205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204442.268216:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204442.268219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268223:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204442.268224:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268226:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268228:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204442.268231:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204442.268232:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204442.268233:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204442.268235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268237:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204442.268240:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204442.268241:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204442.268242:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204442.268243:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268244:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204442.268245:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204442.268300:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204442.268305:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204442.268309:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788204442.460328:0:256122:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788204442.460373:0:256122:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788204442.460383:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204442.460387:0:256122:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788204442.460393:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204442.460397:0:256122:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788204442.460402:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204442.460406:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:2.0:1788204442.460411:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204442.460415:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204442.460428:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6198 00000400:00000010:2.0:1788204442.460457:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6198. 00000100:00000001:2.0:1788204442.460463:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204442.460474:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39180 x1875055543428224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460505:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204442.460510:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543428224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460518:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204442.460520:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204442.460530:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204442.460532:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:2.0:1788204442.460535:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204442.460539:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204442.460542:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:2.0:1788204442.460544:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:2.0:1788204442.460547:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204442.460553:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3ad80 x1875055543428352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460561:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204442.460564:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543428352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460570:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204442.460572:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204442.460578:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204442.460580:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000400:00000200:2.0:1788204442.460582:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204442.460585:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204442.460588:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6220 00000400:00000010:2.0:1788204442.460590:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6220. 00000100:00000001:2.0:1788204442.460592:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204442.460598:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a300 x1875055543428480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460605:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204442.460609:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543428480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460615:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204442.460617:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204442.460622:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204442.460624:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000400:00000200:2.0:1788204442.460627:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204442.460630:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204442.460632:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6a18 00000400:00000010:2.0:1788204442.460634:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6a18. 00000100:00000001:2.0:1788204442.460637:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204442.460641:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38a80 x1875055543428608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460648:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204442.460651:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543428608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460657:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204442.460659:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204442.460663:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204442.460666:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:2.0:1788204442.460668:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204442.460671:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204442.460674:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6e58 00000400:00000010:2.0:1788204442.460676:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6e58. 00000100:00000001:2.0:1788204442.460678:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204442.460683:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a680 x1875055543428736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460703:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204442.460706:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543428736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204464 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204442.460712:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204442.460714:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1788204443.292187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204443.292192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204443.292200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204443.292204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204443.292208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204443.292211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204443.292217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204443.292219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204444.316198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204444.316202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204444.316209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204444.316213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204444.316219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204444.316222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204444.316235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204444.316238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204444.316297:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204444.316302:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204444.316305:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204444.509492:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204444.509496:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1788204444.509819:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204444.509822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1788204444.510138:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204444.510140:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1788204444.510412:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204444.510415:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00000800:00000200:1.1:1788204444.510677:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204444.510679:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 57 secs later) 00010000:00000001:0.0:1788204445.340191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204445.340195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204445.340200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204445.340202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204445.340207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204445.340208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204445.340210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204445.340213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204445.340271:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204445.340274:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204445.340278:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204445.852207:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320804614, at 4320804672 00000100:00000010:2.0:1788204445.852218:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16478a0. 00010000:00000001:0.0:1788204446.364299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204446.364302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204446.364307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204446.364308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204446.364311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204446.364312:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204446.364315:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204446.364316:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204447.388199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204447.388206:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204447.388209:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204447.388214:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204447.388219:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204447.388221:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204447.388224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204447.388228:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204447.388234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204447.388238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204447.388242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204447.388245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204447.388249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204447.388250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204447.388314:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204447.388318:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204447.388321:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204448.412171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204448.412176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204448.412183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204448.412186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204448.412192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204448.412195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204448.412210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204448.412213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204448.412262:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204448.412272:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204448.412275:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204449.436189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204449.436194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204449.436203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204449.436207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204449.436212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204449.436216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204449.436224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204449.436228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204449.436287:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204449.436291:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204449.436294:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204449.628772:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204449.628776:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204449.629399:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204449.629401:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204449.629681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204449.629683:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204449.629945:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204449.629946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204449.630206:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204449.630208:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00010000:00000001:0.0:1788204450.460204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204450.460210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204450.460218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204450.460221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204450.460236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204450.460239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204450.460245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204450.460248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204450.460332:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204450.460336:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204450.460340:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204450.972306:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320809733, at 4320809792 00000100:00000010:2.0:1788204450.972312:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c00. 00010000:00000001:0.0:1788204451.484179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204451.484184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204451.484200:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204451.484204:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204451.484210:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484221:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204451.484224:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484227:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204451.484237:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204451.484238:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204451.484239:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204451.484242:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484244:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484245:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204451.484250:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204451.484257:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204451.484259:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204451.484262:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484263:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204451.484265:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204451.484334:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204451.484338:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204451.484341:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204452.508186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204452.508190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204452.508196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204452.508199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204452.508204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204452.508206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204452.508211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204452.508213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204452.508275:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204452.508279:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204452.508282:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204453.532173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204453.532176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204453.532182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204453.532184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204453.532187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204453.532190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204453.532195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204453.532196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204453.532263:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204453.532267:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204453.532270:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204454.556265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204454.556277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204454.556287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204454.556294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204454.556305:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204454.556310:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204454.556323:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204454.556328:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204454.556396:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204454.556399:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204454.556402:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204454.748664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204454.748670:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204454.749275:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204454.749276:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204454.749447:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204454.749448:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204454.749604:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204454.749606:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204454.749873:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204454.749875:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00010000:00000001:0.0:1788204455.580247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204455.580254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204455.580262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204455.580265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204455.580272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204455.580277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204455.580285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204455.580290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204455.580439:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204455.580444:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204455.580447:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204456.092225:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320814853, at 4320814912 00000100:00000010:2.0:1788204456.092241:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00010000:00000001:0.0:1788204456.604237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204456.604244:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204456.604249:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204456.604254:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204456.604260:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204456.604264:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204456.604268:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204456.604276:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204456.604285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204456.604290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204456.604296:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204456.604297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204456.604301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204456.604303:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204456.604367:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204456.604372:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204456.604376:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204457.628192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204457.628196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204457.628203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204457.628206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204457.628210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204457.628212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204457.628219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204457.628223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204457.628274:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204457.628277:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204457.628279:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204458.652212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204458.652217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204458.652225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204458.652229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204458.652241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204458.652245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204458.652252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204458.652256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204458.652313:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204458.652317:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204458.652320:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204459.676197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204459.676203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204459.676222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204459.676227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204459.676235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204459.676239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204459.676245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204459.676247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204459.676289:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204459.676292:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204459.676294:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204459.868176:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204459.868184:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204459.868186:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204459.868189:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204459.868192:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204459.868197:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26146 00000400:00000200:2.0:1788204459.868205:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204459.868210:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204459.868212:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204459.868214:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868217:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868218:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868220:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204459.868223:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204459.868224:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204459.868226:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204459.868229:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204459.868231:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204459.868233:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204459.868234:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868236:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868237:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868238:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204459.868240:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204459.868242:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868244:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204459.868246:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204459.868248:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204459.868250:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868251:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204459.868253:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204459.868259:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204459.868261:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204459.868270:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204459.868271:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868273:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204459.868275:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204459.868278:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204459.868284:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204459.868290:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204459.868298:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543430784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204459.868305:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204459.868309:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204459.868312:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543430784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204459.868327:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204459.868329:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204459.868332:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204459.868333:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204459.868335:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204459.868337:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204459.868340:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26146 00000400:00000200:2.0:1788204459.868344:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204459.868347:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204459.868350:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204459.868353:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868357:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868368:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868373:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204459.868379:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204459.868381:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204459.868384:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204459.868386:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204459.868389:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204459.868392:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204459.868397:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868400:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868403:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868405:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204459.868408:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204459.868416:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868420:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204459.868428:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204459.868436:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204459.868440:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868444:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204459.868448:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204459.868456:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868457:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204459.868461:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204459.868466:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204459.868469:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204459.868471:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868476:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204459.868477:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204459.868480:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204459.868485:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204459.868489:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204459.868497:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543430912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204459.868506:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204459.868510:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204459.868516:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543430912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204459.868525:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204459.868528:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204459.868533:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204459.868535:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204459.868537:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204459.868541:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204459.868546:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26146 00000400:00000200:2.0:1788204459.868551:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204459.868557:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204459.868560:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204459.868569:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868573:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868577:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868580:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204459.868583:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204459.868584:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204459.868585:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204459.868588:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204459.868592:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204459.868593:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204459.868595:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868597:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868600:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868604:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204459.868607:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204459.868612:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868616:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204459.868617:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204459.868619:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204459.868622:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868625:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204459.868628:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204459.868630:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868631:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204459.868636:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204459.868638:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204459.868640:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204459.868642:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868644:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204459.868646:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204459.868649:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204459.868652:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204459.868656:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204459.868662:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543431040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1788204459.868664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204459.868670:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1788204459.868681:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204459.868684:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204459.868692:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543431040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204459.868705:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204459.868710:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204459.868717:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204459.868721:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204459.868723:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204459.868728:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204459.868732:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26146 00000400:00000200:2.0:1788204459.868740:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204459.868745:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204459.868748:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204459.868752:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868754:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868758:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868760:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204459.868763:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204459.868765:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204459.868768:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204459.868770:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204459.868774:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204459.868777:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204459.868779:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868783:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868786:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868788:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204459.868790:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204459.868793:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868795:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204459.868798:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204459.868800:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204459.868804:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868808:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204459.868811:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204459.868813:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868814:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204459.868818:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204459.868819:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204459.868823:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204459.868824:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868826:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204459.868827:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204459.868828:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204459.868831:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204459.868836:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204459.868849:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543431168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204459.868863:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204459.868867:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204459.868874:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543431168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204459.868892:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204459.868896:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204459.868902:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204459.868905:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204459.868908:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204459.868912:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204459.868917:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26146 00000400:00000200:2.0:1788204459.868924:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204459.868929:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204459.868931:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204459.868936:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868941:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868944:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868947:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204459.868951:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204459.868953:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204459.868955:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204459.868957:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204459.868960:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204459.868962:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204459.868967:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868971:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204459.868973:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868977:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204459.868980:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204459.868984:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204459.868987:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204459.868990:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204459.868992:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204459.868995:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.868996:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204459.868999:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204459.869003:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.869005:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204459.869009:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204459.869014:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204459.869016:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204459.869019:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204459.869021:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204459.869023:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204459.869025:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204459.869029:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204459.869033:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204459.869039:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543431296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204459.869050:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204459.869055:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204459.869064:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543431296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204459.869087:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204459.869115:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26156) 00000800:00000200:1.1:1788204459.869304:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204459.869306:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204459.869484:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204459.869486:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204459.869671:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204459.869673:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204459.869861:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204459.869863:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00010000:00000001:0.0:1788204460.700196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204460.700200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204460.700207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204460.700210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204460.700215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204460.700219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204460.700222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204460.700226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204460.700280:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204460.700283:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204460.700287:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204461.212187:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320819973, at 4320820032 00000100:00000010:2.0:1788204461.212200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647900. 00010000:00000001:0.0:1788204461.724182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204461.724186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204461.724196:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204461.724200:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204461.724204:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724209:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204461.724212:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724216:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724222:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204461.724228:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204461.724229:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204461.724229:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724230:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204461.724231:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724232:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724234:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204461.724238:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204461.724239:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204461.724241:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204461.724243:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724244:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204461.724245:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204461.724294:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204461.724297:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204461.724299:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204462.748184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204462.748188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204462.748195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204462.748198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204462.748203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204462.748207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204462.748213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204462.748217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204462.748281:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204462.748285:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204462.748287:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204463.772169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204463.772174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204463.772180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204463.772182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204463.772189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204463.772191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204463.772196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204463.772199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204463.772250:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204463.772255:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204463.772258:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204464.796193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204464.796198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204464.796208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204464.796212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204464.796218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204464.796221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204464.796226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204464.796230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204464.796295:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204464.796299:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204464.796309:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204464.988182:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204464.988191:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204464.988194:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204464.988196:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204464.988200:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204464.988205:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26151 00000400:00000200:0.0:1788204464.988213:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204464.988219:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204464.988221:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204464.988224:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988226:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988228:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988230:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204464.988234:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204464.988235:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204464.988237:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204464.988240:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204464.988241:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204464.988242:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204464.988244:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988245:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988246:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988247:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204464.988249:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204464.988252:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988253:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204464.988255:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204464.988257:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204464.988258:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988260:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204464.988261:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204464.988268:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204464.988270:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204464.988273:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204464.988274:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988276:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204464.988278:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204464.988281:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204464.988287:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204464.988292:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204464.988299:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543431424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204464.988313:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204464.988319:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204464.988322:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543431424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204464.988335:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204464.988337:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204464.988339:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204464.988340:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204464.988341:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204464.988343:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204464.988344:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26151 00000400:00000200:0.0:1788204464.988347:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204464.988349:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204464.988350:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204464.988352:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988354:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988356:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988357:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204464.988358:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204464.988359:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204464.988360:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204464.988362:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204464.988364:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204464.988366:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204464.988368:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988371:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988374:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988376:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204464.988378:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204464.988382:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988384:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204464.988387:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204464.988392:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204464.988395:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988397:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204464.988401:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204464.988404:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988405:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204464.988407:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204464.988410:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204464.988414:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204464.988417:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988421:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204464.988423:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204464.988425:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204464.988430:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204464.988433:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204464.988439:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543431552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204464.988447:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204464.988450:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204464.988453:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543431552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204464.988463:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204464.988466:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204464.988470:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204464.988473:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204464.988475:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204464.988477:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204464.988481:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26151 00000400:00000200:0.0:1788204464.988484:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204464.988485:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204464.988487:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204464.988491:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988493:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988494:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988496:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204464.988500:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204464.988502:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204464.988505:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204464.988507:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204464.988509:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204464.988512:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204464.988514:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988517:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988520:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988523:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204464.988534:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204464.988541:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988544:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204464.988546:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204464.988549:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204464.988551:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988552:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204464.988554:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204464.988557:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988559:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204464.988562:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204464.988564:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204464.988566:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204464.988568:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988571:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204464.988572:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204464.988574:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204464.988578:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204464.988582:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204464.988587:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543431680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204464.988596:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204464.988598:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204464.988603:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543431680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204464.988612:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204464.988615:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204464.988619:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204464.988622:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204464.988626:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204464.988630:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204464.988635:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26151 00000400:00000200:0.0:1788204464.988642:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204464.988647:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204464.988652:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204464.988657:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988661:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988666:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988669:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204464.988673:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204464.988675:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204464.988677:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204464.988680:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204464.988684:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204464.988686:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204464.988691:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988693:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988696:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.1:1788204464.988697:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204464.988698:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204464.988702:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204464.988705:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000200:1.1:1788204464.988705:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00000001:0.0:1788204464.988708:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204464.988711:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204464.988713:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204464.988716:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988718:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204464.988721:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204464.988723:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988727:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204464.988730:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204464.988732:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204464.988735:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204464.988736:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988738:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204464.988741:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204464.988743:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204464.988748:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204464.988750:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204464.988757:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543431808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204464.988762:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204464.988766:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204464.988771:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543431808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204464.988795:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204464.988798:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204464.988801:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204464.988803:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204464.988805:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204464.988808:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204464.988812:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26151 00000400:00000200:0.0:1788204464.988818:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204464.988823:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204464.988827:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204464.988831:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988833:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988835:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988837:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204464.988841:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204464.988842:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204464.988843:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204464.988846:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204464.988849:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204464.988852:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204464.988856:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988858:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204464.988859:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988861:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204464.988865:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204464.988866:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204464.988868:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204464.988870:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204464.988873:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204464.988875:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988876:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204464.988876:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204464.988878:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988879:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204464.988882:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204464.988886:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204464.988888:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204464.988890:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204464.988893:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204464.988894:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204464.988897:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204464.988901:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204464.988905:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204464.988910:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543431936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204464.988916:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204464.988918:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204464.988925:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543431936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204464.988934:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204464.988938:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26161) 00000800:00000200:1.1:1788204464.989245:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204464.989248:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204464.989510:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204464.989513:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204464.989744:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204464.989746:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204464.989973:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204464.989975:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00010000:00000001:0.0:1788204465.820180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204465.820184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204465.820193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204465.820198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204465.820205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204465.820209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204465.820217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204465.820221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204465.820271:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204465.820274:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204465.820276:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204466.332170:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320825093, at 4320825152 00000100:00000010:2.0:1788204466.332192:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00010000:00000001:0.0:1788204466.844197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204466.844204:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204466.844210:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204466.844217:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204466.844224:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204466.844229:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204466.844236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204466.844242:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204466.844248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204466.844249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204466.844254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204466.844255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204466.844259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204466.844260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204466.844316:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204466.844319:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204466.844322:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204467.868167:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204467.868171:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204467.868177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204467.868180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204467.868186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204467.868189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204467.868197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204467.868199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204467.868254:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204467.868260:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204467.868264:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204468.892204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204468.892210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204468.892220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204468.892224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204468.892239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204468.892243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204468.892248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204468.892249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204468.892311:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204468.892315:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204468.892318:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204469.916170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204469.916175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204469.916181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204469.916184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204469.916189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204469.916191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204469.916195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204469.916198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204469.916261:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204469.916265:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204469.916268:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204470.108791:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204470.108796:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204470.109453:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204470.109455:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204470.109621:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204470.109622:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204470.109810:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204470.109812:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204470.110108:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204470.110110:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00010000:00000001:0.0:1788204470.940190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204470.940195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204470.940205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204470.940209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204470.940216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204470.940221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204470.940226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204470.940229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204470.940291:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204470.940294:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204470.940296:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204471.452178:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320830213, at 4320830272 00000100:00000010:2.0:1788204471.452194:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc5a0. 00010000:00000001:0.0:1788204471.964190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204471.964195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204471.964203:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204471.964206:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204471.964209:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204471.964218:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964225:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964228:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204471.964240:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204471.964242:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204471.964245:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964249:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204471.964253:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964256:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964260:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204471.964265:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204471.964266:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204471.964268:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964270:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204471.964272:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204471.964276:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204471.964325:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204471.964328:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204471.964330:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204472.988323:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204472.988325:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204472.988329:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204472.988330:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204472.988332:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204472.988333:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204472.988335:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204472.988336:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204472.988385:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204472.988387:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204472.988389:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204474.012194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204474.012200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204474.012208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204474.012212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204474.012218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204474.012221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204474.012227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204474.012230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204474.012279:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204474.012283:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204474.012286:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204475.036182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204475.036187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204475.036191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204475.036195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204475.036201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204475.036203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204475.036207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204475.036209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204475.036260:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204475.036264:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204475.036266:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204475.228730:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204475.228735:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204475.229324:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204475.229327:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204475.229556:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204475.229557:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204475.229743:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204475.229746:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204475.230027:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204475.230029:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1788204476.060189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204476.060194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204476.060201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204476.060204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204476.060207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204476.060209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204476.060212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204476.060213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204476.060240:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204476.060245:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204476.060249:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204476.572179:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320835333, at 4320835392 00000100:00000010:2.0:1788204476.572195:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcde0. 00010000:00000001:0.0:1788204477.084183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204477.084189:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204477.084194:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204477.084198:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204477.084205:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204477.084208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204477.084211:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204477.084214:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204477.084218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204477.084219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204477.084222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204477.084224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204477.084227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204477.084228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204477.084281:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204477.084285:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204477.084288:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204478.108195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204478.108200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204478.108207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204478.108210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204478.108217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204478.108221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204478.108224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204478.108227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204478.108284:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204478.108288:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204478.108291:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204479.132194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204479.132198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204479.132205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204479.132208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204479.132212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204479.132214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204479.132219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204479.132222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204479.132278:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204479.132280:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204479.132283:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204480.156188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204480.156192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204480.156201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204480.156210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204480.156218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204480.156220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204480.156225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204480.156228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204480.156279:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204480.156282:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204480.156285:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204480.348792:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204480.348800:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204480.349378:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204480.349381:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204480.349658:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204480.349661:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204480.349967:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204480.349970:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204480.350264:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204480.350266:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788204481.180186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204481.180189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204481.180196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204481.180198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204481.180204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204481.180207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204481.180214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204481.180216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204481.180272:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204481.180275:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204481.180277:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204481.692248:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320840453, at 4320840512 00000100:00000010:2.0:1788204481.692269:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00010000:00000001:0.0:1788204482.204211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204482.204216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204482.204229:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204482.204235:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204482.204241:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204247:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204482.204251:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204261:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204482.204266:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204482.204267:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204482.204268:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204269:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204482.204269:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204270:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204272:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204482.204276:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204482.204277:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204482.204278:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204279:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204482.204280:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204281:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204482.204282:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204482.204352:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204482.204357:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204482.204360:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204483.228171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204483.228176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204483.228181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204483.228184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204483.228191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204483.228194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204483.228199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204483.228204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204483.228300:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204483.228308:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204483.228312:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204484.252183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204484.252187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204484.252193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204484.252196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204484.252200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204484.252202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204484.252208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204484.252212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204484.252298:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204484.252304:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204484.252308:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204485.276212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204485.276220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204485.276226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204485.276230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204485.276238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204485.276242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204485.276248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204485.276256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204485.276339:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204485.276344:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204485.276348:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204485.468272:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204485.468285:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204485.468289:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204485.468294:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204485.468299:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204485.468308:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26171 00000400:00000200:0.0:1788204485.468319:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204485.468326:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204485.468329:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204485.468332:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468334:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.468336:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468337:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204485.468341:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204485.468343:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204485.468346:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204485.468349:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204485.468350:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204485.468352:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204485.468354:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468355:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.468356:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468357:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204485.468359:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204485.468363:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468365:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204485.468368:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204485.468371:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204485.468373:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468375:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204485.468378:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204485.468388:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204485.468391:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204485.468395:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204485.468396:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468399:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204485.468402:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204485.468407:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204485.468416:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204485.468422:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204485.468428:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543433984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204485.468436:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204485.468440:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204485.468443:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543433984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204485.468462:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204485.468464:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204485.468467:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204485.468468:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204485.468469:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204485.468471:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204485.468474:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26171 00000400:00000200:0.0:1788204485.468476:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204485.468479:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204485.468480:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204485.468483:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468484:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.468486:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468488:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204485.468490:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204485.468491:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204485.468492:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204485.468493:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204485.468495:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204485.468496:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204485.468501:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468502:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.468506:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468509:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204485.468511:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204485.468515:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468520:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204485.468522:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204485.468529:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204485.468531:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468534:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204485.468538:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204485.468542:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468547:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204485.468551:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204485.468555:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204485.468558:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204485.468560:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468561:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204485.468564:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204485.468566:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204485.468571:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204485.468575:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204485.468580:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543434112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204485.468589:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204485.468592:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204485.468596:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543434112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204485.468602:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204485.468604:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204485.468607:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204485.468608:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204485.468609:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204485.468611:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204485.468615:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26171 00000400:00000200:0.0:1788204485.468622:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204485.468628:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204485.468632:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204485.468636:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468640:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.468643:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468645:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204485.468651:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204485.468654:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204485.468657:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204485.468659:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204485.468661:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204485.468664:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204485.468670:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468676:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.468681:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468694:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204485.468700:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204485.468705:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468711:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204485.468716:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204485.468719:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204485.468724:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468727:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204485.468731:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204485.468735:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468737:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204485.468742:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204485.468746:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204485.468752:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204485.468753:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468754:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204485.468756:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204485.468758:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204485.468763:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204485.468769:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204485.468777:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543434240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204485.468784:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204485.468786:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204485.468789:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543434240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204485.468800:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204485.468805:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204485.468808:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204485.468811:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1788204485.468812:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204485.468813:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204485.468818:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1788204485.468819:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00080000:0.0:1788204485.468823:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26171 00000400:00000200:0.0:1788204485.468830:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204485.468836:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204485.468840:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204485.468844:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468849:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.468853:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468857:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204485.468861:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204485.468863:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204485.468865:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204485.468869:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204485.468874:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204485.468877:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204485.468882:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468886:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.468890:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468893:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204485.468898:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204485.468902:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204485.468906:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204485.468907:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204485.468909:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204485.468912:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468914:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204485.468917:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204485.468919:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468922:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204485.468927:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204485.468930:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204485.468936:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204485.468938:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.468941:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204485.468943:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204485.468946:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204485.468949:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204485.468952:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204485.468960:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543434368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204485.468968:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204485.468971:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204485.468974:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543434368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204485.468981:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204485.468983:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204485.468986:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204485.468988:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204485.468991:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204485.468993:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204485.468995:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26171 00000400:00000200:0.0:1788204485.468999:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204485.469004:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204485.469005:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204485.469010:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.469012:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.469015:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.469017:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204485.469020:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204485.469022:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204485.469024:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204485.469027:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204485.469030:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204485.469032:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204485.469036:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.469037:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204485.469040:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204485.469042:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204485.469043:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204485.469044:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204485.469048:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204485.469051:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204485.469054:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204485.469058:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.469059:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204485.469061:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204485.469064:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.469068:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204485.469075:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204485.469081:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204485.469087:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204485.469130:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204485.469135:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204485.469138:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204485.469142:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204485.469148:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204485.469154:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204485.469163:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543434496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204485.469177:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204485.469182:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204485.469190:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543434496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204485.469204:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204485.469212:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26181) 00000800:00000200:1.1:1788204485.469387:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204485.469389:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204485.469561:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204485.469562:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204485.469770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204485.469772:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204485.469987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204485.469990:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204486.300372:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204486.300376:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204486.300379:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204486.300380:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204486.300382:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204486.300383:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204486.300385:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204486.300386:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204486.300454:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204486.300459:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204486.300463:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204486.812249:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320845573, at 4320845632 00000100:00000010:2.0:1788204486.812264:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. 00010000:00000001:0.0:1788204487.324184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204487.324193:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204487.324198:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204487.324205:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204487.324211:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204487.324216:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204487.324230:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204487.324233:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204487.324239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204487.324241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204487.324244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204487.324245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204487.324249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204487.324250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204487.324290:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204487.324294:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204487.324296:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204488.348226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204488.348233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204488.348243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204488.348251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204488.348256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204488.348258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204488.348262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204488.348263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204488.348338:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204488.348343:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204488.348347:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204489.372234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204489.372238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204489.372243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204489.372244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204489.372248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204489.372250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204489.372253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204489.372254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204489.372315:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204489.372319:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204489.372322:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204490.396411:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204490.396416:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204490.396422:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204490.396423:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204490.396426:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204490.396427:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204490.396431:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204490.396433:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204490.396493:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204490.396498:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204490.396501:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204490.588840:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204490.588846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788204490.589363:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204490.589365:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788204490.589538:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204490.589539:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788204490.589739:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204490.589741:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:3.1:1788204490.589909:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204490.589910:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00010000:00000001:0.0:1788204491.420224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204491.420234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204491.420246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204491.420249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204491.420254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204491.420256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204491.420260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204491.420261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204491.420325:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204491.420330:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204491.420334:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204491.932220:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320850693, at 4320850752 00000100:00000010:2.0:1788204491.932240:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcba0. 00010000:00000001:0.0:1788204492.444161:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204492.444165:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204492.444174:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204492.444178:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204492.444183:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444187:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204492.444190:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444192:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444198:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204492.444208:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204492.444212:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204492.444214:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444217:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204492.444218:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444220:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444222:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204492.444229:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204492.444230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204492.444232:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444233:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204492.444235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204492.444237:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204492.444266:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204492.444269:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204492.444271:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204493.468186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204493.468192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204493.468200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204493.468203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204493.468208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204493.468211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204493.468220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204493.468223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204493.468266:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204493.468269:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204493.468271:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204494.492187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204494.492193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204494.492202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204494.492205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204494.492209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204494.492213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204494.492220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204494.492224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204494.492280:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204494.492284:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204494.492287:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204495.516175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204495.516180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204495.516186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204495.516188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204495.516191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204495.516195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204495.516198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204495.516202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204495.516240:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204495.516243:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204495.516252:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204495.708232:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204495.708242:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204495.708246:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204495.708249:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204495.708253:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204495.708261:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26182 00000400:00000200:2.0:1788204495.708271:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204495.708278:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204495.708281:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204495.708284:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708287:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.708289:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708292:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204495.708296:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204495.708298:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204495.708301:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204495.708304:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204495.708307:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204495.708309:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204495.708312:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708314:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.708316:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708318:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204495.708321:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204495.708324:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708327:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204495.708329:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204495.708332:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204495.708334:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.708335:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204495.708337:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204495.708345:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204495.708348:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204495.708351:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204495.708352:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.708354:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204495.708356:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204495.708360:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204495.708366:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204495.708371:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204495.708382:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543435264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204495.708392:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204495.708397:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204495.708402:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543435264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204495.708425:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204495.708429:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204495.708433:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204495.708434:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204495.708436:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204495.708439:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204495.708442:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26182 00000400:00000200:2.0:1788204495.708447:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204495.708450:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204495.708453:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204495.708456:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708458:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.708460:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708463:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204495.708465:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204495.708468:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204495.708471:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204495.708474:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204495.708478:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204495.708482:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204495.708499:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708504:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.708539:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708544:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204495.708548:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204495.708554:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708559:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204495.708563:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204495.708569:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204495.708573:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.708577:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204495.708583:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204495.708589:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.708590:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204495.708595:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204495.708600:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204495.708606:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204495.708610:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.708615:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204495.708617:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204495.708620:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204495.708627:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204495.708634:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204495.708642:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543435392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204495.708657:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204495.708662:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204495.708668:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543435392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204495.708682:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204495.708687:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204495.708693:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204495.708697:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204495.708701:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204495.708706:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204495.708721:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26182 00000400:00000200:2.0:1788204495.708743:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204495.708750:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204495.708754:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204495.708758:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708763:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.708775:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788204495.708777:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204495.708780:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788204495.708783:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000020:00000001:2.0:1788204495.708784:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204495.708787:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204495.708792:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204495.708796:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204495.708802:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204495.708806:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204495.708812:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708822:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.708828:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708832:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204495.708837:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204495.708842:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204495.708848:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204495.708852:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204495.708857:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204495.708862:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.708867:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204495.708871:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204495.708885:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.708893:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204495.708900:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204495.708906:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204495.708910:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204495.708913:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.708916:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204495.708918:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204495.708920:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204495.708940:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204495.708946:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204495.708952:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543435520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204495.708964:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204495.708969:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204495.708978:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543435520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204495.708992:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204495.708996:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204495.709003:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204495.709006:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204495.709010:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204495.709015:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204495.709021:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26182 00000400:00000200:2.0:1788204495.709030:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204495.709037:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204495.709042:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204495.709048:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709053:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.709059:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709063:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204495.709068:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204495.709072:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204495.709075:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204495.709078:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204495.709084:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204495.709087:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204495.709137:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709143:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.709148:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709152:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204495.709158:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204495.709164:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709170:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204495.709175:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204495.709179:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204495.709184:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.709188:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204495.709193:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204495.709198:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.709202:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204495.709209:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204495.709215:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204495.709219:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204495.709221:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.709222:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204495.709224:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204495.709226:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204495.709233:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204495.709240:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204495.709249:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543435648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204495.709262:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204495.709265:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204495.709270:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543435648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204495.709284:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204495.709288:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204495.709293:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204495.709295:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204495.709300:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204495.709306:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204495.709311:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26182 00000400:00000200:2.0:1788204495.709319:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204495.709324:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204495.709330:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204495.709334:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709336:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.709341:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709344:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204495.709348:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204495.709351:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204495.709355:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204495.709359:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204495.709363:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204495.709368:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204495.709375:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709377:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204495.709380:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709384:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204495.709390:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204495.709395:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204495.709397:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204495.709403:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204495.709406:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204495.709410:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.709412:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204495.709416:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204495.709418:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.709422:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204495.709426:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204495.709430:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204495.709436:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204495.709439:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204495.709443:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204495.709446:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204495.709449:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204495.709455:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204495.709461:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204495.709472:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543435776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204495.709491:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204495.709496:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204495.709515:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543435776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204495.709531:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204495.709538:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26192) 00000800:00000200:3.1:1788204495.709574:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204495.709576:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204495.709863:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204495.709866:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204495.710155:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204495.710158:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:3.1:1788204495.710551:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204495.710553:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00010000:00000001:0.0:1788204496.540180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204496.540187:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204496.540193:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204496.540197:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204496.540208:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204496.540210:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204496.540213:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204496.540215:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204496.540218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204496.540220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204496.540223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204496.540224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204496.540227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204496.540229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204496.540264:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204496.540269:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204496.540276:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204497.052190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320855813, at 4320855872 00000100:00000010:2.0:1788204497.052203:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fca20. 00010000:00000001:0.0:1788204497.564203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204497.564209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204497.564218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204497.564223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204497.564229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204497.564232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204497.564237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204497.564239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204497.564298:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204497.564302:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204497.564305:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204498.588244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204498.588255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204498.588268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204498.588275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204498.588284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204498.588294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204498.588309:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204498.588316:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204498.588377:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204498.588380:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204498.588382:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204499.612228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204499.612234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204499.612242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204499.612245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204499.612252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204499.612256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204499.612260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204499.612263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204499.612334:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204499.612339:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204499.612342:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204500.636236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204500.636242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204500.636252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204500.636256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204500.636262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204500.636265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204500.636272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204500.636275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204500.636350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204500.636355:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204500.636359:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204500.828232:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204500.828243:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204500.828246:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204500.828250:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204500.828255:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204500.828264:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26187 00000400:00000200:0.0:1788204500.828275:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204500.828282:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204500.828286:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204500.828290:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828293:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.828296:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828298:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204500.828302:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204500.828304:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204500.828307:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204500.828312:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204500.828314:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204500.828317:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204500.828320:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828323:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.828325:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828327:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204500.828330:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204500.828333:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828335:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204500.828338:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204500.828342:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204500.828345:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828348:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204500.828351:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204500.828362:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204500.828366:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204500.828371:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204500.828372:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828376:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204500.828380:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204500.828386:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204500.828400:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788204500.828406:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204500.828413:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543435904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204500.828452:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204500.828457:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204500.828463:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543435904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204500.828487:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204500.828490:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204500.828492:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204500.828493:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204500.828494:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204500.828497:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204500.828499:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26187 00000400:00000200:0.0:1788204500.828502:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204500.828504:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204500.828506:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204500.828508:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828509:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.828510:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828512:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204500.828514:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204500.828515:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204500.828516:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204500.828518:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204500.828520:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204500.828522:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204500.828524:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828525:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.828527:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828528:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204500.828529:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204500.828531:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828533:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204500.828534:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204500.828541:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204500.828544:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828546:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204500.828551:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204500.828554:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828557:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204500.828561:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204500.828564:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204500.828567:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204500.828569:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828572:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204500.828574:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204500.828577:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204500.828581:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788204500.828585:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204500.828593:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543436032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204500.828603:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204500.828608:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204500.828614:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543436032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204500.828632:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204500.828638:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204500.828643:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204500.828645:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204500.828647:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204500.828651:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204500.828655:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26187 00000400:00000200:0.0:1788204500.828659:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204500.828661:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204500.828664:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204500.828669:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828672:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.828675:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828678:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204500.828680:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204500.828680:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204500.828681:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204500.828683:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204500.828686:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204500.828689:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204500.828693:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828694:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.828696:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828698:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204500.828703:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204500.828706:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828709:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204500.828710:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204500.828713:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204500.828715:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828716:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204500.828719:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204500.828721:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828722:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204500.828727:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204500.828731:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204500.828735:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204500.828737:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828739:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204500.828741:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204500.828743:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204500.828750:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788204500.828756:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204500.828763:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543436160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204500.828773:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204500.828777:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204500.828785:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543436160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204500.828801:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204500.828806:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204500.828813:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204500.828816:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204500.828819:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204500.828822:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204500.828825:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26187 00000400:00000200:0.0:1788204500.828831:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204500.828836:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204500.828837:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204500.828839:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828840:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.828842:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828844:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204500.828847:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204500.828849:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204500.828851:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204500.828853:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204500.828854:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204500.828856:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204500.828861:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828864:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.828865:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828867:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204500.828869:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204500.828873:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204500.828874:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204500.828884:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204500.828888:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204500.828897:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828901:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204500.828906:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204500.828911:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828914:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204500.828918:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204500.828923:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204500.828928:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204500.828931:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.828935:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204500.828937:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204500.828940:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204500.828946:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204500.828951:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204500.828960:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543436288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204500.828972:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204500.828976:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204500.828981:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:0.0:1788204500.828982:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543436288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204500.828989:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000100:00000001:0.0:1788204500.828994:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204500.829000:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204500.829005:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204500.829009:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204500.829011:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204500.829014:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204500.829018:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26187 00000400:00000200:0.0:1788204500.829025:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204500.829033:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204500.829036:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204500.829049:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.829053:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.829058:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.829063:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204500.829067:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204500.829070:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204500.829073:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204500.829077:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204500.829081:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204500.829085:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204500.829129:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.829134:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204500.829139:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204500.829143:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204500.829146:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204500.829149:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204500.829154:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204500.829158:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204500.829163:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204500.829167:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.829170:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204500.829173:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204500.829175:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.829176:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204500.829182:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204500.829185:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204500.829188:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204500.829192:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204500.829196:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204500.829200:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204500.829204:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204500.829215:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204500.829222:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204500.829229:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543436416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204500.829240:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204500.829245:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204500.829254:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543436416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204500.829276:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204500.829283:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26197) 00000800:00000200:3.1:1788204500.829506:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204500.829508:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204500.829707:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204500.829708:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204500.829896:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204500.829898:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:3.1:1788204500.830062:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204500.830064:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00010000:00000001:0.0:1788204501.660190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204501.660196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204501.660204:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204501.660207:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204501.660213:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204501.660218:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660221:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660226:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204501.660240:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204501.660243:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204501.660245:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660247:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204501.660249:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660253:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204501.660260:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204501.660261:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204501.660262:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660264:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204501.660265:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660267:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204501.660268:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204501.660354:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204501.660362:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204501.660366:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204502.172231:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320860934, at 4320860992 00000100:00000010:2.0:1788204502.172247:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fca80. 00010000:00000001:0.0:1788204502.684190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204502.684195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204502.684200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204502.684203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204502.684217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204502.684220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204502.684226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204502.684228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204502.684336:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204502.684344:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204502.684349:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204503.708199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204503.708204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204503.708213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204503.708215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204503.708221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204503.708224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204503.708228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204503.708229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204503.708314:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204503.708321:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204503.708325:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204504.732185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204504.732190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204504.732199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204504.732202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204504.732210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204504.732213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204504.732219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204504.732222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204504.732294:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204504.732299:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204504.732302:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204505.756255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204505.756263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204505.756274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204505.756280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204505.756285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204505.756291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204505.756303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204505.756308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204505.756378:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204505.756383:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204505.756386:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204505.948893:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204505.949320:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204505.949565:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204505.949720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204505.949951:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204506.780175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204506.780181:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204506.780185:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204506.780190:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204506.780195:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204506.780198:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204506.780200:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204506.780203:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204506.780207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204506.780209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204506.780212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204506.780214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204506.780217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204506.780218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204506.780273:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204506.780276:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204506.780279:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204507.292190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320866053, at 4320866112 00000100:00000010:2.0:1788204507.292205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc420. 00010000:00000001:0.0:1788204507.804239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204507.804247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204507.804258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204507.804268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204507.804280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204507.804287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204507.804311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204507.804317:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204507.804378:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204507.804382:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204507.804385:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204508.828220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204508.828226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204508.828235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204508.828239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204508.828247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204508.828251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204508.828256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204508.828259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204508.828337:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204508.828342:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204508.828346:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788204509.020517:0:256123:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:0.0:1788204509.020575:0:256123:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204509.020588:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204509.020593:0:256123:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:0.0:1788204509.020601:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204509.020605:0:256123:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:0.0:1788204509.020611:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204509.020616:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:0.0:1788204509.020622:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204509.020627:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204509.020635:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734660 00000400:00000010:0.0:1788204509.020638:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734660. 00000100:00000001:0.0:1788204509.020645:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204509.020654:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7d180 x1875055543436544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021168:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204509.021176:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543436544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021187:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.021189:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204509.021205:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204509.021207:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdb00. 00000400:00000200:0.0:1788204509.021212:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204509.021217:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204509.021224:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987344c8 00000400:00000010:0.0:1788204509.021227:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987344c8. 00000100:00000001:0.0:1788204509.021231:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204509.021236:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7f480 x1875055543436672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021245:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204509.021248:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543436672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021255:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.021256:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204509.021262:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204509.021265:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdd00. 00000400:00000200:0.0:1788204509.021268:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204509.021271:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204509.021286:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734bb0 00000400:00000010:0.0:1788204509.021289:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734bb0. 00000100:00000001:0.0:1788204509.021292:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204509.021297:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7d880 x1875055543436800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021306:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204509.021308:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543436800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021314:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.021315:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204509.021322:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204509.021325:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:0.0:1788204509.021328:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204509.021331:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204509.021335:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734088 00000400:00000010:0.0:1788204509.021337:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734088. 00000100:00000001:0.0:1788204509.021340:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204509.021345:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7fb80 x1875055543436928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021353:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204509.021356:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543436928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021363:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.021364:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204509.021371:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204509.021373:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd900. 00000400:00000200:0.0:1788204509.021376:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204509.021379:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204509.021382:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734d48 00000400:00000010:0.0:1788204509.021385:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734d48. 00000100:00000001:0.0:1788204509.021387:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204509.021396:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7e300 x1875055543437056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021404:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204509.021410:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543437056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204530 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204509.021415:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204509.021416:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1788204509.852198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204509.852203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204509.852212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204509.852216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204509.852221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204509.852225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204509.852240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204509.852243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204509.852330:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204509.852335:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204509.852338:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204510.876186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204510.876189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204510.876196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204510.876200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204510.876205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204510.876208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204510.876214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204510.876217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204510.876275:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204510.876278:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204510.876280:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204511.068277:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204511.068288:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204511.068291:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204511.068294:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204511.068298:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204511.068305:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26197 00000400:00000200:0.0:1788204511.068315:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204511.068322:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204511.068325:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204511.068329:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068332:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068334:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068336:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204511.068340:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204511.068342:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204511.068344:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204511.068348:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204511.068350:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204511.068352:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204511.068354:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068357:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068358:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068361:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204511.068363:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204511.068366:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068369:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204511.068373:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204511.068376:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204511.068379:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068382:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204511.068384:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204511.068391:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204511.068395:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204511.068398:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204511.068400:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068404:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204511.068406:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204511.068411:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204511.068417:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204511.068424:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204511.068433:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543437184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204511.068444:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204511.068449:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204511.068453:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543437184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204511.068491:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204511.068493:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204511.068497:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204511.068499:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204511.068500:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204511.068503:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204511.068506:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26197 00000400:00000200:0.0:1788204511.068510:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204511.068514:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204511.068516:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204511.068519:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068521:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068523:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068525:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204511.068527:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204511.068529:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204511.068530:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204511.068533:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204511.068535:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204511.068537:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204511.068540:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068542:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068544:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068545:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204511.068547:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204511.068550:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068552:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204511.068554:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204511.068559:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204511.068561:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068563:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204511.068567:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204511.068570:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068572:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204511.068575:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204511.068578:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204511.068580:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204511.068580:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068582:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204511.068582:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204511.068584:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204511.068586:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204511.068590:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204511.068595:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543437312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204511.068603:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204511.068605:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204511.068609:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543437312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204511.068616:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204511.068619:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204511.068623:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204511.068624:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204511.068626:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204511.068628:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204511.068630:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26197 00000400:00000200:0.0:1788204511.068635:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204511.068637:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204511.068640:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204511.068642:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068644:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068646:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068648:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204511.068650:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204511.068651:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204511.068653:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204511.068655:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204511.068656:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204511.068658:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204511.068661:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068663:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068664:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068666:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204511.068669:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204511.068671:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068673:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204511.068675:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204511.068677:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204511.068680:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068681:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204511.068684:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204511.068686:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068687:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204511.068691:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204511.068693:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204511.068695:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204511.068696:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068698:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204511.068699:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204511.068701:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204511.068704:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204511.068713:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204511.068718:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543437440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204511.068724:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204511.068726:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204511.068729:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543437440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204511.068735:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204511.068738:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204511.068741:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204511.068742:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204511.068743:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204511.068746:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204511.068749:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26197 00000400:00000200:0.0:1788204511.068753:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204511.068755:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204511.068757:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204511.068760:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068761:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068763:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068765:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204511.068766:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204511.068767:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204511.068769:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204511.068770:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204511.068772:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204511.068774:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204511.068776:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068778:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068779:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068781:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204511.068783:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204511.068785:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068787:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204511.068788:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204511.068790:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204511.068792:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068793:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204511.068795:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204511.068797:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068798:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204511.068801:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204511.068804:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204511.068806:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204511.068807:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068808:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204511.068810:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204511.068811:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204511.068814:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204511.068817:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204511.068821:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543437568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204511.068827:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204511.068829:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204511.068832:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543437568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204511.068839:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204511.068841:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204511.068843:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204511.068845:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204511.068846:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204511.068849:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204511.068851:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26197 00000400:00000200:0.0:1788204511.068854:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204511.068856:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204511.068858:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204511.068861:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068862:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068864:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068866:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204511.068868:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204511.068869:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204511.068870:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204511.068872:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204511.068874:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204511.068876:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204511.068879:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068880:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204511.068882:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068884:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204511.068886:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204511.068888:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204511.068890:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204511.068892:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204511.068895:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204511.068896:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068898:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204511.068900:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204511.068902:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068903:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204511.068906:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204511.068908:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204511.068911:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204511.068912:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204511.068914:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204511.068915:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204511.068916:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204511.068920:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204511.068922:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204511.068926:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543437696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204511.068933:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204511.068934:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204511.068937:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543437696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204511.068945:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204511.068950:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26207) 00000800:00000200:3.1:1788204511.069518:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204511.069521:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204511.071190:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204511.071193:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204511.071517:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204511.071519:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204511.071771:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204511.071773:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788204511.072014:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204511.072017:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204511.900194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204511.900199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204511.900211:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204511.900215:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204511.900222:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900240:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204511.900246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900252:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900259:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204511.900268:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204511.900269:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204511.900271:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900273:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204511.900275:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900276:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900278:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204511.900284:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204511.900287:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204511.900288:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900290:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204511.900292:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900293:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204511.900296:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204511.900354:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204511.900357:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204511.900361:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204512.412195:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320871174, at 4320871232 00000100:00000010:2.0:1788204512.412208:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc3c0. 00010000:00000001:0.0:1788204512.924185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204512.924189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204512.924195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204512.924199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204512.924204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204512.924219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204512.924226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204512.924229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204512.924290:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204512.924295:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204512.924298:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204513.948228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204513.948231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204513.948238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204513.948241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204513.948244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204513.948246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204513.948250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204513.948252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204513.948309:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204513.948313:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204513.948316:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204514.972210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204514.972213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204514.972222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204514.972223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204514.972230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204514.972233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204514.972237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204514.972239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204514.972297:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204514.972302:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204514.972306:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204515.996188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204515.996193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204515.996201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204515.996203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204515.996209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204515.996212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204515.996216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204515.996220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204515.996288:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204515.996292:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204515.996295:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204516.188211:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204516.188222:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204516.188224:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204516.188228:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204516.188232:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204516.188239:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26202 00000400:00000200:0.0:1788204516.188250:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204516.188256:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204516.188259:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204516.188262:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188265:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.188268:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188270:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204516.188274:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204516.188276:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204516.188280:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204516.188283:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204516.188285:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204516.188288:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204516.188290:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188292:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.188294:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188296:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204516.188299:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204516.188302:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188304:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204516.188307:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204516.188310:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204516.188313:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.188315:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204516.188318:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204516.188327:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204516.188330:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204516.188341:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204516.188343:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.188346:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204516.188349:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204516.188352:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204516.188361:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204516.188367:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204516.188377:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543437824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204516.188388:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204516.188392:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204516.188397:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543437824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204516.188425:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204516.188428:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204516.188432:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204516.188433:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204516.188435:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204516.188437:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204516.188490:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26202 00000400:00000200:0.0:1788204516.188500:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204516.188503:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204516.188505:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204516.188511:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188515:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.188519:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188522:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204516.188525:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204516.188528:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204516.188531:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204516.188535:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204516.188539:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204516.188544:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204516.188550:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188555:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.188559:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188563:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204516.188567:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204516.188573:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188580:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204516.188586:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204516.188593:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204516.188597:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.188602:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204516.188607:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204516.188613:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.188617:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204516.188623:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204516.188628:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204516.188633:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204516.188636:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.188640:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204516.188643:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204516.188646:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204516.188653:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204516.188660:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204516.188667:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543437952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204516.188679:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204516.188684:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204516.188691:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543437952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204516.188706:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204516.188711:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204516.188715:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204516.188716:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204516.188718:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204516.188720:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204516.188724:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26202 00000400:00000200:0.0:1788204516.188732:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204516.188740:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204516.188743:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204516.188747:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188752:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.188757:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188761:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204516.188766:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204516.188769:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204516.188772:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204516.188775:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204516.188780:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204516.188784:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000800:00000200:3.1:1788204516.188789:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:0.0:1788204516.188792:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188795:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.188799:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788204516.188800:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1788204516.188803:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204516.188809:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204516.188815:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204516.188822:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204516.188827:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204516.188831:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204516.188835:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.188840:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204516.188842:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204516.188846:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.188849:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204516.188854:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204516.188859:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204516.188864:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204516.188867:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.188870:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204516.188874:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204516.188878:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204516.188898:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204516.188904:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204516.188912:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543438080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204516.188923:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204516.188926:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204516.188930:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543438080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204516.188945:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204516.188949:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204516.188957:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204516.188961:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204516.188962:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204516.188967:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204516.188972:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26202 00000400:00000200:0.0:1788204516.188981:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204516.188985:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204516.188990:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204516.188995:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189001:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.189005:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189010:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204516.189014:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204516.189016:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204516.189019:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204516.189022:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204516.189026:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204516.189031:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204516.189039:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189042:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.189047:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189052:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204516.189056:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204516.189063:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189069:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204516.189073:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204516.189077:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204516.189081:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.189084:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204516.189089:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204516.189121:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.189126:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204516.189131:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204516.189134:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204516.189139:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204516.189142:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.189146:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204516.189148:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204516.189160:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204516.189167:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204516.189172:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204516.189180:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543438208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204516.189196:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204516.189201:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204516.189209:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543438208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204516.189223:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204516.189228:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204516.189235:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204516.189237:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204516.189241:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204516.189246:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204516.189251:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26202 00000400:00000200:0.0:1788204516.189256:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204516.189259:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204516.189262:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204516.189267:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189271:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.189275:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189279:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204516.189284:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204516.189287:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204516.189290:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204516.189292:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204516.189296:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204516.189301:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204516.189305:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189308:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204516.189314:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189317:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204516.189322:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204516.189329:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204516.189336:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204516.189339:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204516.189342:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204516.189346:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.189351:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204516.189354:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204516.189360:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.189365:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204516.189368:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204516.189372:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204516.189377:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204516.189380:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204516.189383:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204516.189386:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204516.189390:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204516.189395:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204516.189402:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204516.189410:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543438336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204516.189425:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204516.189430:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204516.189437:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543438336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204516.189451:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204516.189458:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26212) 00000800:00000200:3.1:1788204516.189533:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204516.189536:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204516.189770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204516.189771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204516.189989:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204516.189991:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204516.190330:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204516.190332:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788204517.020198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204517.020205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204517.020209:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204517.020212:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204517.020216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204517.020219:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204517.020223:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204517.020227:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204517.020232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204517.020234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204517.020237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204517.020239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204517.020242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204517.020243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204517.020293:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204517.020296:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204517.020299:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204517.532221:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320876293, at 4320876352 00000100:00000010:2.0:1788204517.532227:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc9c0. 00010000:00000001:0.0:1788204518.044227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204518.044231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204518.044236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204518.044239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204518.044243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204518.044246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204518.044253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204518.044255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204518.044316:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204518.044320:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204518.044322:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204519.068206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204519.068212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204519.068220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204519.068223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204519.068230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204519.068233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204519.068241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204519.068245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204519.068343:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204519.068346:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204519.068349:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204520.092207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204520.092212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204520.092220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204520.092224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204520.092231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204520.092235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204520.092242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204520.092245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204520.092306:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204520.092309:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204520.092312:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204521.116192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204521.116197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204521.116203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204521.116207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204521.116213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204521.116216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204521.116223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204521.116227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204521.116289:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204521.116293:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204521.116296:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204521.308871:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204521.308876:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204521.309498:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204521.309500:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204521.309773:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204521.309775:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204521.310043:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204521.310045:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204521.310339:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204521.310342:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1788204522.140172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204522.140175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204522.140185:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204522.140188:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204522.140191:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140195:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204522.140199:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140202:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140205:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204522.140209:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204522.140210:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204522.140211:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140212:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204522.140214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140216:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204522.140222:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204522.140223:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204522.140225:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140226:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204522.140227:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140228:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204522.140230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204522.140283:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204522.140288:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204522.140291:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204522.652201:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320881413, at 4320881472 00000100:00000010:2.0:1788204522.652216:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647540. 00010000:00000001:0.0:1788204523.164212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204523.164217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204523.164226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204523.164229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204523.164236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204523.164239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204523.164244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204523.164248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204523.164322:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204523.164326:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204523.164329:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204524.188170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204524.188176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204524.188181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204524.188184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204524.188190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204524.188193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204524.188199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204524.188201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204524.188260:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204524.188265:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204524.188268:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204525.212194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204525.212199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204525.212208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204525.212213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204525.212219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204525.212222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204525.212230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204525.212234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204525.212299:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204525.212303:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204525.212307:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204526.236191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204526.236195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204526.236204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204526.236208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204526.236217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204526.236220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204526.236228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204526.236231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204526.236258:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204526.236262:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204526.236265:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204526.428226:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204526.428242:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204526.428245:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204526.428248:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204526.428252:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204526.428259:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26212 00000400:00000200:2.0:1788204526.428269:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204526.428277:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204526.428280:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204526.428284:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428286:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428288:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428289:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204526.428293:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204526.428294:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204526.428297:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204526.428300:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204526.428302:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204526.428305:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204526.428307:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428309:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428310:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428312:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204526.428315:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204526.428326:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428328:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204526.428331:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204526.428334:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204526.428337:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428339:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204526.428342:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204526.428350:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204526.428353:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204526.428355:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204526.428356:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428359:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204526.428360:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204526.428364:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204526.428370:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204526.428375:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204526.428383:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543439104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204526.428390:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204526.428393:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204526.428396:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543439104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204526.428413:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204526.428415:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204526.428418:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204526.428419:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204526.428420:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204526.428422:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204526.428423:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26212 00000400:00000200:2.0:1788204526.428426:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204526.428428:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204526.428429:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204526.428431:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428433:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428434:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428436:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204526.428438:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204526.428438:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204526.428439:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204526.428441:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204526.428442:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204526.428444:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204526.428446:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428449:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428451:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428455:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204526.428458:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204526.428462:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428467:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204526.428470:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204526.428475:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204526.428479:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428482:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204526.428486:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204526.428489:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428492:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204526.428495:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204526.428499:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204526.428502:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204526.428504:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428507:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204526.428509:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204526.428512:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204526.428514:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204526.428517:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204526.428523:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543439232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204526.428530:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204526.428532:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204526.428538:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543439232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204526.428545:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204526.428548:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204526.428552:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204526.428554:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204526.428557:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204526.428558:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204526.428560:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26212 00000400:00000200:2.0:1788204526.428566:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204526.428568:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204526.428570:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204526.428574:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428577:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428579:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428582:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204526.428584:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204526.428587:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204526.428589:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204526.428592:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204526.428596:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204526.428598:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204526.428605:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428609:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428611:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428615:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204526.428616:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204526.428621:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428625:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204526.428626:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204526.428628:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204526.428629:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428630:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204526.428632:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204526.428634:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428638:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204526.428640:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204526.428642:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204526.428645:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204526.428648:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428651:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204526.428653:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204526.428655:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204526.428660:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204526.428663:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204526.428668:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543439360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204526.428674:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204526.428675:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204526.428678:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543439360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204526.428685:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204526.428688:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204526.428692:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204526.428700:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204526.428702:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204526.428706:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204526.428711:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26212 00000400:00000200:2.0:1788204526.428718:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204526.428723:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204526.428727:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204526.428729:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428731:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428734:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428736:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204526.428737:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204526.428738:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204526.428739:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204526.428742:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204526.428744:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204526.428747:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204526.428750:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428751:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428752:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428753:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204526.428756:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204526.428758:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428761:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204526.428765:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204526.428767:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204526.428770:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204526.428770:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204526.428774:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.1:1788204526.428775:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000008:00100000:2.0:1788204526.428776:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204526.428780:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428783:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204526.428788:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204526.428791:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204526.428793:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204526.428796:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428799:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204526.428800:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204526.428801:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204526.428803:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204526.428806:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204526.428811:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543439488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204526.428820:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204526.428823:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204526.428828:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543439488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204526.428857:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204526.428860:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204526.428867:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204526.428871:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204526.428874:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204526.428881:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204526.428894:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26212 00000400:00000200:2.0:1788204526.428904:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204526.428907:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204526.428909:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204526.428911:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428912:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428914:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428915:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204526.428916:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204526.428917:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204526.428918:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204526.428919:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204526.428924:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204526.428925:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204526.428927:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428929:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204526.428932:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428935:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204526.428937:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204526.428941:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204526.428943:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204526.428946:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204526.428949:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204526.428955:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428957:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204526.428959:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204526.428962:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428965:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204526.428968:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204526.428971:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204526.428982:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204526.428983:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204526.428986:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204526.428987:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204526.428990:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204526.428995:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204526.428999:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204526.429004:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543439616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204526.429011:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204526.429014:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204526.429020:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543439616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204526.429029:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204526.429034:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26222) 00000800:00000200:1.1:1788204526.429268:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204526.429269:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204526.429554:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204526.429556:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204526.429831:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204526.429834:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204526.430141:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204526.430143:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1788204527.260186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204527.260193:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204527.260197:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204527.260201:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204527.260206:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204527.260209:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204527.260212:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204527.260215:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204527.260221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204527.260225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204527.260230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204527.260233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204527.260240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204527.260243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204527.260305:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204527.260309:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204527.260312:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204527.772216:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320886533, at 4320886592 00000100:00000010:2.0:1788204527.772236:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00010000:00000001:0.0:1788204528.284223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204528.284229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204528.284240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204528.284243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204528.284251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204528.284255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204528.284259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204528.284260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204528.284328:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204528.284332:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204528.284334:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204529.308179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204529.308192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204529.308200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204529.308204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204529.308210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204529.308213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204529.308219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204529.308222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204529.308293:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204529.308298:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204529.308301:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204530.332193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204530.332198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204530.332208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204530.332212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204530.332220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204530.332224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204530.332231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204530.332234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204530.332332:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204530.332336:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204530.332338:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204531.356193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204531.356197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204531.356202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204531.356204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204531.356208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204531.356210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204531.356214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204531.356216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204531.356279:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204531.356282:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204531.356285:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204531.548203:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204531.548212:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204531.548214:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204531.548217:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204531.548221:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204531.548228:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26217 00000400:00000200:2.0:1788204531.548240:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204531.548247:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204531.548251:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204531.548254:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548256:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.548258:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548260:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204531.548263:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204531.548264:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204531.548266:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204531.548269:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204531.548270:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204531.548272:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204531.548274:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548275:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.548276:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548277:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204531.548278:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204531.548280:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548282:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204531.548284:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204531.548287:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204531.548289:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548291:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204531.548294:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204531.548302:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204531.548306:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204531.548309:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204531.548311:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548314:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204531.548317:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204531.548320:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204531.548327:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204531.548334:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204531.548345:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543439744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204531.548356:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204531.548360:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204531.548364:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543439744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204531.548383:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204531.548386:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204531.548388:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204531.548389:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204531.548390:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204531.548392:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204531.548394:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26217 00000400:00000200:2.0:1788204531.548397:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204531.548399:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204531.548401:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204531.548403:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548404:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.548406:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548408:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204531.548409:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204531.548410:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204531.548412:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204531.548414:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204531.548415:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204531.548416:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204531.548419:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548421:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.548422:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548424:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204531.548426:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204531.548428:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548432:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204531.548434:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204531.548441:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204531.548444:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548446:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204531.548450:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204531.548454:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548457:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204531.548462:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204531.548465:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204531.548468:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204531.548473:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548477:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204531.548478:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204531.548480:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204531.548483:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204531.548487:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204531.548494:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543439872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204531.548507:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204531.548511:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204531.548515:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543439872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204531.548525:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204531.548529:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204531.548533:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204531.548537:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204531.548540:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204531.548545:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204531.548551:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26217 00000400:00000200:2.0:1788204531.548558:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204531.548562:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204531.548566:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204531.548570:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548575:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.548579:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548584:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204531.548588:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204531.548591:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204531.548595:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204531.548598:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204531.548600:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204531.548615:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204531.548622:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548626:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.548632:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548636:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204531.548642:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204531.548648:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548653:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204531.548658:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204531.548662:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204531.548668:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548672:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204531.548675:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204531.548678:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548680:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204531.548685:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204531.548689:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204531.548693:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204531.548695:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548699:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204531.548702:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204531.548704:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204531.548707:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204531.548712:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204531.548720:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543440000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204531.548730:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204531.548734:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204531.548738:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543440000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204531.548745:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204531.548749:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204531.548755:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204531.548757:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204531.548760:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1788204531.548767:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204531.548771:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1788204531.548773:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00080000:2.0:1788204531.548776:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26217 00000400:00000200:2.0:1788204531.548783:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204531.548790:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204531.548794:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204531.548797:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548802:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.548808:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548812:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204531.548817:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204531.548828:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204531.548833:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204531.548837:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204531.548842:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204531.548848:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204531.548856:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548861:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.548867:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548873:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204531.548879:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204531.548897:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204531.548902:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204531.548904:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204531.548910:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204531.548916:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548919:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204531.548923:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204531.548928:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548932:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204531.548936:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204531.548940:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204531.548943:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204531.548945:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.548948:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204531.548949:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204531.548951:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204531.548957:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204531.548962:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204531.548970:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543440128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204531.548979:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204531.548984:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204531.549002:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543440128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204531.549020:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204531.549025:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204531.549036:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204531.549039:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204531.549043:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204531.549051:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204531.549057:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26217 00000400:00000200:2.0:1788204531.549067:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204531.549074:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204531.549080:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204531.549086:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.549117:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.549120:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.549122:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204531.549124:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204531.549125:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204531.549127:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204531.549129:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204531.549132:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204531.549135:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204531.549138:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.549143:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204531.549148:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204531.549154:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204531.549158:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204531.549163:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204531.549166:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204531.549169:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204531.549173:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204531.549177:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.549180:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204531.549183:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204531.549186:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.549190:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204531.549195:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204531.549198:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204531.549202:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204531.549204:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204531.549209:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204531.549211:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204531.549214:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204531.549220:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204531.549226:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204531.549237:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543440256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204531.549252:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204531.549256:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204531.549263:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543440256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204531.549271:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204531.549277:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26228) 00000800:00000200:1.1:1788204531.549434:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204531.549437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204531.549699:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204531.549701:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204531.549968:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204531.549970:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204531.550239:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204531.550241:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00010000:00000001:0.0:1788204532.380197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204532.380202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204532.380225:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204532.380229:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204532.380234:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380238:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204532.380241:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380243:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380246:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204532.380252:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204532.380253:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204532.380255:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380257:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204532.380258:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380259:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380261:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204532.380266:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204532.380267:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204532.380269:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380270:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204532.380271:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380273:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204532.380275:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204532.380332:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204532.380336:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204532.380340:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204532.892173:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320891653, at 4320891712 00000100:00000010:2.0:1788204532.892191:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647000. 00010000:00000001:0.0:1788204533.404191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204533.404198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204533.404203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204533.404206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204533.404208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204533.404210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204533.404213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204533.404216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204533.404287:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204533.404291:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204533.404294:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204534.428190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204534.428198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204534.428205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204534.428209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204534.428214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204534.428218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204534.428224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204534.428226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204534.428227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204534.428232:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204534.428234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204534.428236:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204534.428238:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204534.428240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204534.428241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204534.428243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204534.428245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.452176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204535.452182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204535.452186:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204535.452190:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204535.452191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204535.452196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.452197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204535.452203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204535.452204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204535.452206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204535.452211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204535.452211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204535.452215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.452216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204535.452217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204535.452220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204535.452226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204536.476200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204536.476208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204536.476211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204536.476215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204536.476216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204536.476220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204536.476222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204536.476225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204536.476227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204536.476228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204536.476231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204536.476234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204536.476235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204536.476237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204536.476240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204536.476243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204536.476245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204536.476268:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204536.476272:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204536.476276:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204536.668195:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204536.668203:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204536.668205:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204536.668208:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204536.668211:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204536.668217:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26223 00000400:00000200:0.0:1788204536.668226:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204536.668231:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204536.668233:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204536.668235:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668238:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668240:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668242:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204536.668246:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204536.668247:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204536.668250:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204536.668253:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204536.668256:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204536.668258:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204536.668261:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668263:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668264:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668266:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204536.668269:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204536.668272:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668274:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204536.668276:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204536.668279:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204536.668282:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668284:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204536.668287:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204536.668296:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204536.668299:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204536.668303:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204536.668304:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668307:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204536.668310:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204536.668314:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204536.668321:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204536.668328:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204536.668338:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543440384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204536.668348:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204536.668351:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204536.668355:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543440384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204536.668372:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204536.668375:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204536.668377:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204536.668378:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204536.668379:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204536.668381:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204536.668383:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26223 00000400:00000200:0.0:1788204536.668386:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204536.668388:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204536.668389:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204536.668391:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668392:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668393:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668395:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204536.668396:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204536.668398:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204536.668399:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204536.668400:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204536.668402:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204536.668403:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204536.668405:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668406:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668408:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668409:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204536.668410:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204536.668412:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668414:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204536.668415:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204536.668420:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204536.668422:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668425:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204536.668428:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204536.668431:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668434:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204536.668440:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204536.668441:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204536.668444:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204536.668445:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668447:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204536.668449:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204536.668451:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204536.668455:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204536.668459:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204536.668466:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543440512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204536.668481:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204536.668484:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204536.668490:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543440512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204536.668499:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204536.668502:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204536.668506:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204536.668507:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204536.668508:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204536.668512:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204536.668515:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26223 00000400:00000200:0.0:1788204536.668519:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204536.668522:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204536.668524:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204536.668525:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668540:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668544:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668548:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204536.668551:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204536.668552:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204536.668555:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204536.668558:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204536.668561:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204536.668564:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204536.668570:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668573:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668576:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668579:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204536.668583:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204536.668586:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668590:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204536.668593:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204536.668597:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204536.668601:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668605:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204536.668606:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204536.668610:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668613:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204536.668617:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204536.668621:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204536.668623:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204536.668625:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668628:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204536.668629:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204536.668630:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204536.668635:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204536.668639:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204536.668643:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543440640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204536.668653:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204536.668655:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204536.668660:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543440640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204536.668667:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204536.668671:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204536.668675:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204536.668678:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204536.668680:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204536.668683:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204536.668688:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26223 00000400:00000200:0.0:1788204536.668704:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204536.668710:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204536.668713:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204536.668718:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668722:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668724:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668726:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204536.668729:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204536.668731:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204536.668733:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204536.668735:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204536.668736:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204536.668738:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204536.668743:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668745:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668748:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668751:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204536.668755:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204536.668758:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668760:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204536.668762:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204536.668764:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204536.668766:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668768:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204536.668770:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000800:00000200:1.1:1788204536.668771:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000008:00000001:0.0:1788204536.668773:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668776:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204536.668778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000010:0.0:1788204536.668780:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204536.668785:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204536.668790:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204536.668819:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668823:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204536.668826:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204536.668829:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204536.668836:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204536.668840:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204536.668848:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543440768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204536.668864:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204536.668868:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204536.668873:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543440768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204536.668880:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204536.668883:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204536.668891:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204536.668893:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204536.668895:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204536.668900:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204536.668903:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26223 00000400:00000200:0.0:1788204536.668910:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204536.668913:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204536.668915:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204536.668917:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668921:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668924:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668926:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204536.668931:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204536.668931:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204536.668933:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204536.668935:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204536.668938:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204536.668940:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204536.668941:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668943:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204536.668946:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668948:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204536.668950:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204536.668952:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204536.668953:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204536.668957:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204536.668960:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204536.668961:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668963:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204536.668964:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204536.668969:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668971:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204536.668974:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204536.668977:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204536.668979:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204536.668981:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204536.668983:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204536.668984:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204536.668986:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204536.668992:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204536.668996:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204536.669000:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543440896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204536.669007:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204536.669009:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204536.669015:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543440896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204536.669033:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204536.669037:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26233) 00000800:00000200:1.1:1788204536.669350:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204536.669352:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204536.669601:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204536.669603:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204536.669860:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204536.669862:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204536.670151:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204536.670154:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1788204537.500245:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204537.500259:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204537.500263:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204537.500269:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204537.500271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204537.500275:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204537.500275:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204537.500279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204537.500280:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204537.500285:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204537.500290:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204537.500291:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204537.500296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204537.500297:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204537.500302:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204537.500304:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204537.500307:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204537.500311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204537.500313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204537.500317:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204537.500318:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204537.500322:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204537.500323:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204538.012179:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320896773, at 4320896832 00000100:00000010:2.0:1788204538.012192:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00000100:00000001:2.0:1788204538.524194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204538.524206:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204538.524208:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204538.524211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204538.524218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204538.524219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204538.524222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204538.524230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204538.524230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204538.524233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204538.524245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204538.524256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204538.524259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204538.524264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204538.524266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204538.524271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204538.524272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204539.548182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204539.548189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204539.548192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204539.548199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204539.548205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204539.548209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204539.548215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204539.548217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204539.548221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204539.548317:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204539.548320:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204539.548323:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204539.548324:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204539.548327:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204539.548328:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204539.548331:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204539.548333:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204539.548388:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204539.548390:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204539.548392:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204540.572187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204540.572192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204540.572219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204540.572220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204540.572222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204540.572223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204540.572226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204540.572228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204540.572228:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204540.572235:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204540.572239:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204540.572258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204540.572262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204540.572265:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.572268:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204540.572270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204540.572273:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204541.596200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204541.596214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204541.596214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204541.596219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204541.596219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204541.596227:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204541.596228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204541.596232:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204541.596233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204541.596236:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204541.596241:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204541.596241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204541.596245:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204541.596246:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204541.596249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596250:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204541.596253:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596255:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596257:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204541.596261:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204541.596263:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204541.596264:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596265:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204541.596267:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596268:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596269:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204541.596272:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204541.596273:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204541.596274:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596275:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204541.596275:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596276:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204541.596277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204541.788823:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204541.788827:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204541.789516:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204541.789519:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204541.789793:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204541.789795:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204541.790005:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204541.790009:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204541.790224:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204541.790225:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788204542.620185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204542.620190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204542.620194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204542.620197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204542.620197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204542.620204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204542.620206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204542.620211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204542.620211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204542.620216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204542.620218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204542.620220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204542.620221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204542.620225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204542.620227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204542.620230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204542.620233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204543.132189:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320901893, at 4320901952 00000100:00000010:2.0:1788204543.132205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647420. 00000100:00000001:2.0:1788204543.644165:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204543.644171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204543.644173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204543.644175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204543.644177:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204543.644180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204543.644184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204543.644184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204543.644186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204543.644190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204543.644191:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204543.644193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204543.644195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204543.644196:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204543.644201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204543.644206:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204543.644210:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204544.668181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204544.668183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204544.668188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204544.668188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204544.668193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204544.668195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204544.668197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204544.668199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204544.668202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204544.668203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204544.668206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204544.668207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204544.668211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204544.668215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204544.668216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204544.668220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204544.668221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204545.692149:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204545.692157:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204545.692161:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204545.692167:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204545.692171:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204545.692176:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204545.692180:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204545.692184:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204545.692190:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204545.692281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204545.692285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204545.692289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204545.692291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204545.692294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204545.692295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204545.692298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204545.692300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204546.716182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204546.716187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204546.716191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204546.716193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204546.716194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204546.716200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204546.716201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204546.716204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204546.716207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204546.716209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204546.716210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204546.716212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204546.716215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204546.716216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204546.716218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204546.716221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204546.716225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204546.908719:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204546.908722:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204546.909435:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204546.909437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204546.909713:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204546.909714:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204546.909937:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204546.909939:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204546.910264:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204546.910266:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788204547.740183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204547.740189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204547.740192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204547.740192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204547.740198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204547.740199:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204547.740203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204547.740204:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204547.740207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204547.740210:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204547.740210:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204547.740214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204547.740216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204547.740220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204547.740221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204547.740223:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204547.740226:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204547.740234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204547.740238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204547.740242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204547.740243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204547.740248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204547.740250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204548.252185:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320907013, at 4320907072 00000100:00000010:2.0:1788204548.252199:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00000100:00000001:2.0:1788204548.764175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204548.764180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204548.764187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204548.764187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204548.764191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204548.764194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204548.764196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204548.764198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204548.764203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204548.764204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204548.764208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204548.764208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204548.764214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204548.764216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204548.764218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204548.764220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204548.764221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204549.788186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204549.788195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204549.788196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204549.788198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204549.788200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204549.788206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204549.788207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204549.788210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204549.788213:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204549.788214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204549.788217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204549.788219:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204549.788220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204549.788222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204549.788226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204549.788230:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204549.788237:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204550.812237:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204550.812240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204550.812246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204550.812248:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204550.812254:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204550.812259:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204550.812264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204550.812267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204550.812268:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204550.812275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204550.812276:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204550.812281:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204550.812283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204550.812292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204550.812293:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204550.812309:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204550.812312:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204551.836205:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204551.836215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204551.836218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204551.836224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204551.836228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204551.836232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204551.836232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204551.836244:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204551.836245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204551.836253:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204551.836253:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204551.836258:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204551.836259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204551.836263:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204551.836265:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836269:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204551.836272:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836278:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836281:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836288:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204551.836295:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204551.836298:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204551.836300:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836305:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204551.836307:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836319:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836322:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836325:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204551.836327:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204551.836328:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204551.836329:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836329:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204551.836330:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836331:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204551.836332:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204552.028919:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204552.028924:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204552.029672:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204552.029674:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204552.029945:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204552.029947:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204552.030243:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204552.030245:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204552.030523:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204552.030526:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788204552.860220:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204552.860224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204552.860229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204552.860234:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204552.860235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204552.860240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204552.860240:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204552.860252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204552.860252:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204552.860260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204552.860264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204552.860268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204552.860270:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204552.860273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204552.860276:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204552.860280:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204552.860287:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204553.372183:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320912133, at 4320912192 00000100:00000010:2.0:1788204553.372198:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647960. 00000100:00000001:2.0:1788204553.884183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204553.884191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204553.884193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204553.884196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204553.884198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204553.884203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204553.884206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204553.884210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204553.884211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204553.884215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204553.884217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204553.884218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204553.884224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204553.884226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204553.884229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204553.884229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204553.884236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204554.908158:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204554.908163:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204554.908165:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204554.908169:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204554.908174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204554.908175:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204554.908180:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204554.908181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204554.908183:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204554.908187:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204554.908190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204554.908191:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204554.908194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204554.908201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204554.908205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204554.908213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204554.908216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204555.932161:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204555.932163:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204555.932167:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204555.932168:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204555.932173:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204555.932174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204555.932177:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204555.932180:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204555.932181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204555.932183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204555.932186:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204555.932187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204555.932190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204555.932191:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204555.932198:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204555.932203:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204555.932208:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204556.956179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204556.956186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204556.956190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204556.956192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204556.956196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204556.956199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204556.956201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204556.956204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204556.956205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204556.956211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204556.956212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204556.956215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204556.956217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204556.956218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204556.956221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204556.956222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204556.956227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204557.148201:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204557.148209:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204557.148211:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204557.148213:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204557.148217:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204557.148222:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26243 00000400:00000200:0.0:1788204557.148231:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204557.148236:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204557.148238:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204557.148240:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148243:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.148244:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148246:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204557.148249:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204557.148251:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204557.148253:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204557.148256:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204557.148257:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204557.148259:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204557.148261:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148262:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.148263:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148264:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204557.148266:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204557.148268:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148269:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204557.148271:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204557.148273:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204557.148275:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148276:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204557.148278:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204557.148285:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204557.148287:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204557.148290:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204557.148291:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148293:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204557.148295:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204557.148298:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204557.148304:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204557.148309:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204557.148319:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543442944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204557.148329:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204557.148334:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204557.148338:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543442944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204557.148359:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204557.148363:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204557.148366:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204557.148368:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204557.148369:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204557.148372:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204557.148375:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26243 00000400:00000200:0.0:1788204557.148379:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204557.148382:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204557.148385:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204557.148388:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148391:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.148393:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148396:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204557.148398:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204557.148401:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204557.148405:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204557.148409:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204557.148412:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204557.148416:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204557.148422:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148427:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.148431:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148437:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204557.148450:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204557.148456:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148468:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204557.148472:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204557.148478:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204557.148482:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148486:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204557.148491:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204557.148495:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148498:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204557.148502:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204557.148505:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204557.148510:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204557.148512:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148514:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204557.148515:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204557.148517:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204557.148522:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204557.148526:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204557.148534:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543443072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204557.148546:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204557.148549:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204557.148555:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543443072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204557.148566:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204557.148569:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204557.148572:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204557.148574:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204557.148576:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204557.148580:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204557.148585:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26243 00000400:00000200:0.0:1788204557.148591:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204557.148596:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204557.148600:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204557.148606:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148610:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.148612:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148613:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204557.148615:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204557.148615:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204557.148617:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204557.148620:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204557.148622:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204557.148624:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204557.148630:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148632:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.148633:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148634:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204557.148636:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204557.148640:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148643:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204557.148644:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204557.148647:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204557.148651:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148653:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204557.148655:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204557.148658:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148660:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204557.148664:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204557.148669:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204557.148671:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204557.148674:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148679:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204557.148682:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204557.148684:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204557.148689:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204557.148692:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204557.148699:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543443200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204557.148722:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204557.148726:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204557.148734:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543443200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788204557.148743:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204557.148749:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:0.0:1788204557.148753:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204557.148759:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204557.148765:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204557.148770:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204557.148774:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204557.148780:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204557.148786:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26243 00000400:00000200:0.0:1788204557.148797:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204557.148804:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204557.148808:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204557.148813:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148820:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.148827:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148831:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204557.148837:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204557.148840:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204557.148845:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204557.148848:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204557.148852:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204557.148858:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204557.148864:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148870:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.148875:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148879:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204557.148883:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204557.148888:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204557.148893:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204557.148897:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204557.148902:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204557.148907:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148912:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204557.148915:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204557.148921:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148923:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204557.148927:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204557.148931:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204557.148936:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204557.148939:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.148944:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204557.148947:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204557.148950:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204557.148958:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204557.148964:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204557.148975:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543443328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204557.148990:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204557.148994:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204557.149000:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543443328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204557.149019:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204557.149024:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204557.149027:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204557.149029:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204557.149031:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204557.149034:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204557.149038:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26243 00000400:00000200:0.0:1788204557.149046:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204557.149051:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204557.149054:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204557.149059:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.149070:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.149076:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.149080:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204557.149085:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204557.149086:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204557.149117:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204557.149120:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204557.149124:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204557.149129:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204557.149137:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.149140:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204557.149143:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204557.149146:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204557.149149:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204557.149152:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204557.149155:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204557.149157:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204557.149161:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204557.149165:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.149167:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204557.149171:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204557.149175:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.149177:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204557.149182:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204557.149187:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204557.149189:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204557.149191:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204557.149192:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204557.149195:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204557.149199:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204557.149203:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204557.149208:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204557.149216:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543443456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204557.149227:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204557.149230:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204557.149235:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543443456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204557.149251:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204557.149257:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26253) 00000800:00000200:1.1:1788204557.149321:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204557.149323:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204557.149504:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204557.149505:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204557.149710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204557.149712:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204557.149928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204557.149929:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788204557.980186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204557.980191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204557.980193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204557.980197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204557.980197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204557.980200:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204557.980204:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204557.980210:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:2.0:1788204557.980210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204557.980213:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204557.980214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204557.980217:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204557.980217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204557.980222:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204557.980222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204557.980225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204557.980228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204557.980228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204557.980230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204557.980233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204557.980235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204557.980238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204557.980239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204558.492183:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320917253, at 4320917312 00000100:00000010:2.0:1788204558.492200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. 00000100:00000001:2.0:1788204559.004172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204559.004179:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204559.004181:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204559.004186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204559.004188:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204559.004191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204559.004193:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204559.004197:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204559.004199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204559.004200:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204559.004203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204559.004203:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204559.004209:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204559.004210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204559.004214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204559.004218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204559.004220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204560.028180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204560.028187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204560.028188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204560.028190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204560.028193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204560.028197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204560.028200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204560.028203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204560.028204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204560.028209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204560.028211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204560.028214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204560.028214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204560.028218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204560.028221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204560.028221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204560.028224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204561.052182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204561.052185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204561.052189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204561.052190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204561.052194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204561.052196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204561.052199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204561.052200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204561.052204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204561.052207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204561.052207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204561.052213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204561.052214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204561.052216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204561.052220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204561.052224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204561.052228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204562.076190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204562.076197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204562.076199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204562.076202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204562.076204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076206:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204562.076210:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204562.076210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204562.076212:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204562.076215:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204562.076218:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204562.076219:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204562.076222:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204562.076223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204562.076224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204562.076227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204562.076230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204562.076231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204562.076241:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204562.076243:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204562.076245:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076247:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204562.076249:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076254:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204562.076260:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204562.076261:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204562.076264:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076265:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204562.076266:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076268:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204562.076270:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204562.268698:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204562.268702:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204562.269379:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204562.269384:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204562.269567:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204562.269569:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204562.269720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204562.269722:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204562.269987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204562.269989:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788204563.100163:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204563.100171:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204563.100174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204563.100175:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204563.100180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204563.100181:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204563.100187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204563.100187:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204563.100191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204563.100192:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204563.100196:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204563.100197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204563.100201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204563.100201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204563.100205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204563.100210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204563.100212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204563.612181:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320922373, at 4320922432 00000100:00000010:2.0:1788204563.612187:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00000100:00000001:2.0:1788204564.124180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204564.124186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204564.124189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204564.124191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204564.124198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204564.124199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204564.124202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204564.124204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204564.124206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204564.124208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204564.124209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204564.124210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204564.124212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204564.124215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204564.124217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204564.124220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204564.124228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204565.148138:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204565.148145:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204565.148147:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204565.148152:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204565.148155:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204565.148158:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204565.148160:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204565.148162:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204565.148165:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204565.148171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204565.148176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204565.148182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204565.148184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204565.148191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204565.148195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204565.148202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204565.148205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204565.148243:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204565.148245:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204565.148247:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204566.172172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204566.172180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204566.172183:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204566.172188:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204566.172192:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204566.172195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204566.172198:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204566.172202:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204566.172211:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204566.172351:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204566.172354:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204566.172358:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204566.172359:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204566.172362:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204566.172363:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204566.172367:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204566.172368:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204567.196214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204567.196219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204567.196222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.196229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204567.196230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204567.196235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.196240:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204567.196254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204567.196255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204567.196261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.196277:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204567.196280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204567.196282:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204567.196283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.196290:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204567.196296:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204567.196302:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204567.388194:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204567.388203:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204567.388205:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204567.388208:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204567.388212:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204567.388217:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26253 00000400:00000200:2.0:1788204567.388226:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204567.388232:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204567.388234:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204567.388238:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388241:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.388243:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388246:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204567.388250:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204567.388252:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204567.388255:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204567.388258:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204567.388260:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204567.388273:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204567.388276:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388278:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.388280:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388282:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204567.388285:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204567.388289:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388291:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204567.388293:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204567.388296:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204567.388298:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.388299:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204567.388301:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204567.388309:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204567.388312:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204567.388315:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204567.388316:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.388318:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204567.388320:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204567.388323:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204567.388329:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204567.388336:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204567.388343:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543444224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204567.388354:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204567.388359:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204567.388363:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543444224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204567.388385:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204567.388388:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204567.388391:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204567.388392:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204567.388393:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204567.388395:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204567.388397:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26253 00000400:00000200:2.0:1788204567.388400:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204567.388402:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204567.388404:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204567.388406:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388407:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.388410:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388412:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204567.388414:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204567.388415:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204567.388417:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204567.388420:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204567.388422:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204567.388424:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204567.388427:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388430:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.388433:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388436:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204567.388438:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204567.388441:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388445:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204567.388451:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204567.388463:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204567.388471:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.388478:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204567.388483:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204567.388504:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.388520:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204567.388525:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204567.388536:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204567.388548:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204567.388550:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.388559:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204567.388562:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204567.388565:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204567.388573:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204567.388581:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204567.388587:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543444352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204567.388605:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204567.388610:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204567.388616:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543444352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204567.388628:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204567.388637:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204567.388640:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204567.388642:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204567.388647:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204567.388656:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204567.388666:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26253 00000400:00000200:2.0:1788204567.388686:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204567.388699:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204567.388710:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204567.388723:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388729:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.388731:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388740:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204567.388749:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204567.388751:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204567.388753:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204567.388758:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204567.388763:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204567.388774:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204567.388787:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388797:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.388803:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388815:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204567.388822:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204567.388837:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204567.388843:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204567.388847:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204567.388856:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204567.388864:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.388872:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204567.388875:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204567.388880:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204567.388885:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204567.388890:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204567.388890:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000010:2.0:1788204567.388902:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204567.388912:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204567.388919:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204567.388925:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.388929:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204567.388933:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204567.388938:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204567.388941:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204567.388949:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204567.388965:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543444480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204567.388976:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204567.388982:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204567.388993:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543444480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204567.389012:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204567.389018:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204567.389023:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204567.389026:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204567.389030:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204567.389038:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204567.389042:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26253 00000400:00000200:2.0:1788204567.389056:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204567.389062:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204567.389064:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204567.389068:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389077:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.389084:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389117:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204567.389119:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204567.389122:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204567.389124:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204567.389127:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204567.389129:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204567.389132:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204567.389141:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389152:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.389162:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389172:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204567.389176:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204567.389183:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389187:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204567.389192:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204567.389196:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204567.389202:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.389209:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204567.389212:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204567.389219:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.389220:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204567.389224:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204567.389230:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204567.389240:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204567.389242:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.389244:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204567.389246:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204567.389249:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204567.389258:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204567.389277:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204567.389291:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543444608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204567.389310:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204567.389315:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204567.389326:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543444608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204567.389367:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204567.389372:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204567.389379:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204567.389382:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204567.389386:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204567.389396:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204567.389400:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26253 00000400:00000200:2.0:1788204567.389413:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204567.389422:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204567.389428:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204567.389439:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389445:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.389449:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389453:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204567.389457:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204567.389463:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204567.389465:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204567.389469:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204567.389478:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204567.389484:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204567.389496:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389500:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204567.389506:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389520:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204567.389532:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204567.389548:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204567.389560:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204567.389570:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204567.389575:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204567.389579:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.389589:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204567.389591:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204567.389595:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.389610:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204567.389617:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204567.389628:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204567.389631:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204567.389633:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204567.389639:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204567.389646:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204567.389654:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204567.389670:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204567.389674:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204567.389679:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543444736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204567.389705:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204567.389708:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204567.389721:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543444736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204567.389751:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204567.389768:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26263) 00000800:00000200:1.1:1788204567.389844:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204567.389846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204567.390057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204567.390059:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204567.390280:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204567.390282:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204567.390670:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204567.390673:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1788204568.220209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204568.220215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204568.220217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204568.220221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204568.220226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204568.220230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204568.220232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204568.220234:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204568.220239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204568.220242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204568.220260:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204568.220268:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204568.220275:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204568.220283:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204568.220289:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204568.220292:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204568.220295:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204568.220299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204568.220300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204568.220304:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204568.220305:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204568.220307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204568.220308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204568.220348:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204568.220353:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204568.220356:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204568.732171:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320927493, at 4320927552 00000100:00000010:2.0:1788204568.732184:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00000100:00000001:2.0:1788204569.244168:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204569.244171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204569.244175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204569.244176:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204569.244179:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204569.244182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204569.244183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204569.244184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204569.244191:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204569.244192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204569.244195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204569.244195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204569.244201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204569.244202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204569.244206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204569.244207:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204569.244213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204570.268172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204570.268177:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204570.268186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204570.268190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204570.268202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204570.268205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204570.268209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204570.268220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204570.268305:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204570.268308:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204570.268309:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204570.268311:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204570.268312:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204570.268313:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204570.268315:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204570.268316:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204570.268317:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204571.292179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204571.292183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204571.292187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204571.292187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204571.292191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204571.292204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204571.292206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204571.292208:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204571.292211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204571.292213:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204571.292214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204571.292220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204571.292221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204571.292222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204571.292227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204571.292231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204571.292234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204572.316134:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204572.316140:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204572.316142:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204572.316147:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204572.316150:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204572.316153:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204572.316156:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204572.316158:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204572.316160:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204572.316186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204572.316197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204572.316201:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204572.316204:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316208:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204572.316211:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316212:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316214:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204572.316217:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204572.316218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204572.316219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316220:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204572.316221:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316222:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316223:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204572.316227:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204572.316228:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204572.316229:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316230:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204572.316231:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316232:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204572.316234:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204572.508745:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204572.509151:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204572.509454:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204572.509602:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204572.509735:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204573.340201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204573.340201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204573.340208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204573.340210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204573.340214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204573.340216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204573.340221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204573.340222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204573.340225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204573.340228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204573.340229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204573.340233:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204573.340237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204573.340240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204573.340241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204573.340244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204573.340245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204573.852173:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320932613, at 4320932672 00000100:00000010:2.0:1788204573.852189:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00000100:00000001:2.0:1788204574.364137:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204574.364154:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204574.364156:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204574.364162:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204574.364166:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204574.364170:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204574.364173:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204574.364177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204574.364178:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204574.364183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204574.364185:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204574.364191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204574.364193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204574.364197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204574.364200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204574.364205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204574.364207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204575.388196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204575.388199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204575.388203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204575.388203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204575.388206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204575.388208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204575.388210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204575.388211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204575.388213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204575.388216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204575.388218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204575.388219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204575.388221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204575.388222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204575.388226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204575.388229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204575.388232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204576.412186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204576.412194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204576.412195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204576.412197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204576.412200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204576.412204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204576.412208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204576.412211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204576.412212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204576.412217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204576.412219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204576.412223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204576.412224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204576.412229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204576.412231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204576.412235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204576.412235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.436192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204577.436196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204577.436199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204577.436201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.436203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204577.436209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204577.436210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204577.436213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.436215:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204577.436220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204577.436220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204577.436223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.436225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204577.436229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204577.436230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204577.436233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.436234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204577.628199:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204577.628207:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204577.628210:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204577.628212:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204577.628215:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204577.628221:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26264 00000400:00000200:2.0:1788204577.628229:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204577.628236:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204577.628238:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204577.628240:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628243:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.628245:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628247:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204577.628250:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204577.628252:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204577.628254:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204577.628257:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204577.628258:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204577.628260:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204577.628262:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628263:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.628264:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628265:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204577.628267:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204577.628269:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628270:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204577.628272:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204577.628275:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204577.628276:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628278:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204577.628279:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204577.628283:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204577.628285:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204577.628288:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204577.628289:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628291:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204577.628293:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204577.628296:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204577.628301:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204577.628306:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204577.628313:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543445504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204577.628320:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204577.628324:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204577.628328:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543445504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204577.628351:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204577.628355:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204577.628358:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204577.628360:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204577.628361:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204577.628364:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204577.628367:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26264 00000400:00000200:2.0:1788204577.628371:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204577.628374:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204577.628376:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204577.628379:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628381:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.628383:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628385:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204577.628388:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204577.628389:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204577.628391:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204577.628393:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204577.628396:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204577.628400:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204577.628409:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628413:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.628417:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628421:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204577.628425:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204577.628431:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628437:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204577.628441:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204577.628447:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204577.628451:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628455:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204577.628461:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204577.628466:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628470:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204577.628476:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204577.628479:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204577.628483:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204577.628487:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628491:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204577.628494:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204577.628496:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204577.628501:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204577.628506:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204577.628514:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543445632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204577.628526:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204577.628529:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204577.628536:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543445632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204577.628551:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204577.628557:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204577.628565:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204577.628568:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204577.628571:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204577.628575:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204577.628579:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26264 00000400:00000200:2.0:1788204577.628586:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204577.628589:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204577.628594:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204577.628596:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628600:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.628603:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628605:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204577.628608:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204577.628611:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204577.628614:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204577.628616:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204577.628619:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204577.628624:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204577.628630:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628633:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.628638:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628646:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204577.628650:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204577.628657:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628661:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204577.628664:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204577.628669:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204577.628673:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628677:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204577.628680:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204577.628684:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628686:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204577.628692:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204577.628696:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204577.628700:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204577.628701:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628704:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204577.628706:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204577.628708:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204577.628712:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204577.628715:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204577.628723:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543445760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204577.628735:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204577.628739:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204577.628741:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788204577.628746:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543445760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788204577.628748:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788204577.628764:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204577.628770:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204577.628774:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204577.628777:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204577.628780:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204577.628786:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204577.628792:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26264 00000400:00000200:2.0:1788204577.628801:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204577.628807:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204577.628811:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204577.628817:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628821:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.628825:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628829:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204577.628831:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204577.628834:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204577.628837:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204577.628839:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204577.628844:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204577.628849:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204577.628856:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628862:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.628866:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628868:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204577.628870:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204577.628873:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204577.628879:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204577.628881:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204577.628885:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204577.628889:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628893:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204577.628896:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204577.628901:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628902:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204577.628907:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204577.628911:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204577.628914:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204577.628917:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.628921:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204577.628924:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204577.628928:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204577.628935:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204577.628940:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204577.628949:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543445888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204577.628968:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204577.628972:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204577.628977:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543445888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204577.628989:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204577.628993:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204577.628998:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204577.629002:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204577.629004:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204577.629010:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204577.629015:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26264 00000400:00000200:2.0:1788204577.629024:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204577.629030:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204577.629034:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204577.629040:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.629042:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.629046:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.629050:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204577.629054:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204577.629058:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204577.629061:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204577.629063:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204577.629069:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204577.629074:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204577.629081:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.629148:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204577.629154:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204577.629159:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204577.629165:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204577.629171:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204577.629177:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204577.629180:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204577.629185:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204577.629188:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.629193:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204577.629196:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204577.629200:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.629203:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204577.629210:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204577.629214:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204577.629220:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204577.629223:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204577.629227:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204577.629229:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204577.629232:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204577.629238:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204577.629244:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204577.629251:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543446016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204577.629273:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204577.629275:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204577.629281:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543446016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204577.629293:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204577.629300:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26274) 00000800:00000200:1.1:1788204577.629436:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204577.629438:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204577.629697:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204577.629699:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204577.629956:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204577.629958:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204577.630212:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204577.630214:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204578.460239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204578.460247:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204578.460249:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204578.460252:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204578.460256:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204578.460258:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204578.460260:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204578.460262:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204578.460266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204578.460267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204578.460270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204578.460271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204578.460274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204578.460275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204578.462133:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204578.462137:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204578.462138:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204578.462141:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204578.462143:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204578.462145:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204578.462147:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204578.462149:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204578.462150:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204578.972182:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320937733, at 4320937792 00000100:00000010:2.0:1788204578.972202:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00000100:00000001:2.0:1788204579.484183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204579.484185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204579.484190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204579.484192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204579.484194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204579.484198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204579.484198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204579.484204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204579.484205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204579.484206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204579.484213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204579.484214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204579.484222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204579.484226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204579.484230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204579.484234:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204579.484240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204580.508158:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204580.508166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204580.508168:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204580.508172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204580.508173:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204580.508180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204580.508181:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204580.508182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204580.508189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204580.508190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204580.508193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204580.508197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204580.508201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204580.508201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204580.508211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204580.508215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204580.508220:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.532176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204581.532179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204581.532184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.532187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204581.532190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204581.532194:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:2.0:1788204581.532196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204581.532197:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204581.532201:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.532201:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204581.532205:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204581.532205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204581.532208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.532208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204581.532212:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.532213:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204581.532217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204581.532218:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204581.532227:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204581.532228:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204581.532229:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204581.532232:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532235:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204581.532246:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204581.532247:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204581.532249:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532251:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204581.532252:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204581.532255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204582.556164:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204582.556171:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204582.556174:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204582.556175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204582.556181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204582.556182:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204582.556188:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204582.556190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204582.556192:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204582.556194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204582.556196:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204582.556201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204582.556203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204582.556206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204582.556207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204582.556213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204582.556215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204582.748266:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204582.748277:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204582.748279:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204582.748293:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204582.748297:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204582.748304:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26269 00000400:00000200:0.0:1788204582.748315:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204582.748323:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204582.748326:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204582.748330:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748333:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.748335:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748339:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204582.748343:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204582.748345:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204582.748348:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204582.748351:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204582.748353:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204582.748355:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204582.748358:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748360:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.748361:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748362:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204582.748364:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204582.748367:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748368:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204582.748370:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204582.748372:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204582.748374:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.748376:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204582.748378:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204582.748385:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204582.748387:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204582.748390:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204582.748391:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.748393:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204582.748396:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204582.748399:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204582.748405:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788204582.748481:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204582.748493:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543446144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204582.748504:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204582.748509:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204582.748514:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543446144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204582.748537:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204582.748541:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204582.748544:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204582.748546:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204582.748547:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204582.748550:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204582.748553:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26269 00000400:00000200:0.0:1788204582.748557:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204582.748560:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204582.748562:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204582.748565:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748568:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.748570:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748572:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204582.748574:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204582.748576:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204582.748578:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204582.748580:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204582.748582:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204582.748584:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204582.748586:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748587:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.748588:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748590:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204582.748591:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204582.748593:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748596:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204582.748600:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204582.748606:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204582.748611:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.748615:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204582.748620:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204582.748626:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.748630:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204582.748635:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204582.748638:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204582.748644:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204582.748646:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.748648:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204582.748649:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204582.748652:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204582.748657:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788204582.748663:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204582.748673:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543446272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204582.748687:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204582.748691:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204582.748698:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543446272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204582.748712:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204582.748715:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204582.748721:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204582.748722:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204582.748723:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204582.748725:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204582.748728:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26269 00000400:00000200:0.0:1788204582.748736:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204582.748743:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204582.748751:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204582.748759:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748763:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.748767:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748769:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204582.748774:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204582.748777:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204582.748780:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204582.748784:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204582.748787:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204582.748790:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204582.748796:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748801:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.748806:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748808:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204582.748810:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204582.748816:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204582.748820:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204582.748822:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204582.748827:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204582.748831:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.748836:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204582.748839:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204582.748854:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.748858:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204582.748864:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204582.748868:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204582.748872:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204582.748875:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.748878:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204582.748881:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204582.748884:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204582.748890:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788204582.748896:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204582.748904:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543446400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204582.748916:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204582.748920:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204582.748928:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543446400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788204582.748929:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204582.748936:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1788204582.748942:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204582.748948:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204582.748955:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204582.748958:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204582.748961:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204582.748966:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204582.748973:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26269 00000400:00000200:0.0:1788204582.748981:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204582.748989:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204582.748991:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204582.749004:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749009:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.749013:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749015:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204582.749018:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204582.749020:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204582.749022:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204582.749024:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204582.749028:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204582.749032:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204582.749036:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749038:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.749042:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749044:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204582.749047:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204582.749052:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749058:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204582.749062:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204582.749067:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204582.749071:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.749073:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204582.749076:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204582.749126:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.749128:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204582.749132:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204582.749137:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204582.749139:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204582.749140:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.749142:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204582.749146:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204582.749149:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204582.749157:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204582.749164:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204582.749173:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543446528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204582.749187:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204582.749191:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204582.749200:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543446528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204582.749226:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204582.749231:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204582.749237:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204582.749241:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204582.749246:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204582.749252:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204582.749259:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26269 00000400:00000200:0.0:1788204582.749267:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204582.749276:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204582.749281:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204582.749289:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749293:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.749295:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749298:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204582.749301:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204582.749305:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204582.749310:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204582.749315:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204582.749319:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204582.749326:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204582.749334:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749340:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204582.749345:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749352:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204582.749357:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204582.749363:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204582.749368:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204582.749373:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204582.749378:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204582.749382:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.749386:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204582.749390:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204582.749394:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.749398:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204582.749406:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204582.749412:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204582.749416:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204582.749419:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204582.749423:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204582.749426:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204582.749428:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204582.749434:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204582.749439:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204582.749446:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543446656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204582.749459:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204582.749464:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204582.749472:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543446656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204582.749498:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204582.749507:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26279) 00000800:00000200:1.1:1788204582.749618:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204582.749621:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204582.749888:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204582.749892:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204582.750186:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204582.750189:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204582.750549:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204582.750551:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788204583.580195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204583.580197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204583.580203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204583.580204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204583.580209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204583.580212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204583.580214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204583.580217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204583.580220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204583.580225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204583.580225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204583.580229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204583.580231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204583.580235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204583.580236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204583.580239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204583.580243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204584.092194:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320942853, at 4320942912 00000100:00000010:2.0:1788204584.092211:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00000100:00000001:2.0:1788204584.604183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204584.604187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204584.604191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204584.604195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204584.604196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204584.604198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204584.604199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204584.604201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204584.604202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204584.604204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204584.604204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204584.604214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204584.604220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204584.604225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788204584.604227:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000100:00000001:2.0:1788204584.604230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788204584.604232:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204584.604235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1788204584.604239:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204584.604240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204585.628183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204585.628190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204585.628193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204585.628196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204585.628198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204585.628201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204585.628203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204585.628207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204585.628210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204585.628211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204585.628214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204585.628215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204585.628218:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204585.628222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204585.628225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204585.628230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204585.628233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.652178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204586.652185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204586.652186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204586.652189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204586.652198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204586.652200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204586.652204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204586.652207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.652207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204586.652211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204586.652213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204586.652214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204586.652217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204586.652218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204586.652222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204586.652222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204586.652225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204587.676171:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204587.676177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204587.676180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204587.676184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204587.676185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204587.676190:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204587.676192:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204587.676193:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204587.676196:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204587.676197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204587.676201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204587.676202:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204587.676205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204587.676209:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204587.676213:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204587.676215:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204587.676217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204587.676223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204587.676224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204587.676227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204587.676228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204587.676231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204587.676232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204587.676251:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204587.676254:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204587.676256:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204587.868204:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204587.868215:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204587.868217:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204587.868221:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204587.868225:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204587.868239:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26274 00000400:00000200:0.0:1788204587.868250:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204587.868256:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204587.868260:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204587.868263:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868266:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868269:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868271:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204587.868274:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204587.868276:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204587.868279:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204587.868283:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204587.868286:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204587.868288:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204587.868291:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868293:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868295:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868297:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204587.868300:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204587.868303:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868305:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204587.868308:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204587.868311:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204587.868312:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868314:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204587.868316:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204587.868323:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204587.868325:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204587.868328:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204587.868330:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868332:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204587.868334:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204587.868337:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204587.868343:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204587.868348:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204587.868356:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543446784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204587.868362:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204587.868366:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204587.868369:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543446784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204587.868386:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204587.868389:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204587.868391:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204587.868392:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204587.868393:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204587.868395:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204587.868397:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26274 00000400:00000200:0.0:1788204587.868400:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204587.868402:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204587.868403:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204587.868405:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868406:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868407:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868409:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204587.868410:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204587.868411:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204587.868412:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204587.868414:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204587.868415:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204587.868417:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204587.868419:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868421:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868423:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868425:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204587.868427:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204587.868430:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868433:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204587.868438:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204587.868443:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204587.868449:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868453:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204587.868457:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204587.868464:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868468:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204587.868475:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204587.868480:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204587.868485:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204587.868487:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868491:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204587.868493:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204587.868497:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204587.868503:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204587.868509:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204587.868518:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543446912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204587.868529:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204587.868531:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204587.868535:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543446912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204587.868549:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204587.868555:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204587.868561:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204587.868562:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204587.868564:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204587.868570:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204587.868574:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26274 00000400:00000200:0.0:1788204587.868581:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204587.868588:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204587.868591:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204587.868598:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868602:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868606:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868610:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204587.868614:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204587.868616:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204587.868618:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204587.868621:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204587.868624:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204587.868627:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204587.868631:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868633:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868635:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868639:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204587.868642:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204587.868644:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868646:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204587.868650:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204587.868654:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204587.868656:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868657:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204587.868660:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204587.868663:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868667:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204587.868669:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204587.868671:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204587.868674:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204587.868676:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868679:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204587.868680:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204587.868681:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204587.868685:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204587.868689:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204587.868696:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543447040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204587.868705:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204587.868707:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204587.868713:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543447040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204587.868727:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204587.868730:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204587.868732:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204587.868735:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204587.868737:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204587.868742:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204587.868746:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26274 00000400:00000200:0.0:1788204587.868753:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204587.868757:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204587.868761:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204587.868764:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868766:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868767:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868772:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204587.868773:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204587.868774:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204587.868775:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204587.868777:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204587.868779:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204587.868782:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204587.868786:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868788:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868791:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868793:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204587.868795:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000800:00000200:1.1:1788204587.868798:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204587.868799:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868803:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204587.868805:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1788204587.868805:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 02000000:00000001:0.0:1788204587.868806:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204587.868811:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868812:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204587.868815:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204587.868818:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868821:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204587.868824:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204587.868826:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204587.868828:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204587.868829:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868830:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204587.868832:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204587.868833:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204587.868837:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204587.868841:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204587.868847:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543447168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204587.868858:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204587.868861:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204587.868864:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543447168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204587.868871:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204587.868873:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204587.868879:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204587.868881:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204587.868883:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204587.868888:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204587.868893:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26274 00000400:00000200:0.0:1788204587.868896:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204587.868901:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204587.868917:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204587.868923:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868929:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868933:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868937:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204587.868943:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204587.868946:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204587.868947:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204587.868949:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204587.868953:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204587.868959:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204587.868965:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868968:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204587.868971:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868972:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204587.868974:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204587.868978:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204587.868980:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204587.868981:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204587.868983:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204587.868988:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868989:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204587.868990:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204587.868991:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.868993:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204587.868997:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204587.869000:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204587.869004:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204587.869006:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204587.869009:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204587.869020:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204587.869023:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204587.869030:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204587.869036:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204587.869046:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543447296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204587.869059:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204587.869063:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204587.869071:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543447296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204587.869085:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204587.869115:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26284) 00000800:00000200:1.1:1788204587.869386:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204587.869388:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204587.869623:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204587.869625:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204587.869784:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204587.869786:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204587.869961:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204587.869963:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788204588.700183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204588.700186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204588.700190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204588.700192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204588.700196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204588.700200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204588.700201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204588.700204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204588.700206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204588.700208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204588.700210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204588.700211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204588.700217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204588.700217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204588.700221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204588.700221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204588.700228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204589.212184:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320947973, at 4320948032 00000100:00000010:2.0:1788204589.212200:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647720. 00000100:00000001:2.0:1788204589.724187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204589.724190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204589.724195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204589.724195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204589.724199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204589.724204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204589.724204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204589.724208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204589.724210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204589.724215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204589.724216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204589.724220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204589.724221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204589.724226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204589.724227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204589.724232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204589.724233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204590.748179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204590.748180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204590.748186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204590.748188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204590.748192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204590.748193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204590.748194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204590.748199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204590.748200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204590.748201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204590.748207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204590.748208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204590.748211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204590.748212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204590.748217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204590.748221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204590.748225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204590.748238:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204590.748242:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204590.748245:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204591.772156:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204591.772162:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204591.772165:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204591.772165:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772170:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204591.772172:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204591.772174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204591.772178:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204591.772179:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204591.772181:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204591.772182:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204591.772185:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.772185:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204591.772190:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204591.772190:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204591.772193:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204591.772194:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772195:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772197:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204591.772204:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204591.772205:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204591.772209:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772215:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204591.772216:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772219:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772221:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204591.772226:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204591.772227:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204591.772228:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772229:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204591.772230:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772231:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204591.772232:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204592.796187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204592.796188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204592.796194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204592.796195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204592.796197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204592.796201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204592.796203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204592.796208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204592.796209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204592.796213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204592.796215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204592.796219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204592.796220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204592.796222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204592.796226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204592.796228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204592.796229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204592.988778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204592.988786:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204592.989505:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204592.989508:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204592.989769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204592.989772:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204592.990050:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204592.990053:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204592.990399:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204592.990401:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1788204593.820176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204593.820182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204593.820185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204593.820186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204593.820188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204593.820192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204593.820195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204593.820195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204593.820199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204593.820201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204593.820203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204593.820204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204593.820207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204593.820208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204593.820214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204593.820220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788204593.820225:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000100:00000001:2.0:1788204593.820226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204593.820227:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204593.820233:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204594.332197:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320953093, at 4320953152 00000100:00000010:2.0:1788204594.332212:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00000100:00000001:2.0:1788204594.844173:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204594.844180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204594.844183:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204594.844185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204594.844186:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204594.844192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204594.844193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204594.844196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204594.844200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204594.844200:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204594.844204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204594.844205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204594.844207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204594.844210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204594.844212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204594.844216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204594.844220:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204595.868160:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204595.868166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204595.868168:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204595.868171:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204595.868172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204595.868177:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204595.868182:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204595.868186:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204595.868188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204595.868191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204595.868193:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204595.868195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204595.868196:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204595.868197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204595.868200:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204595.868203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204595.868206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.892193:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204596.892199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204596.892202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204596.892206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204596.892207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204596.892212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.892213:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204596.892217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.892220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204596.892221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204596.892226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204596.892226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204596.892232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204596.892234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204596.892239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204596.892243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204596.892245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204596.892265:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204596.892268:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204596.892270:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204597.916151:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204597.916158:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204597.916159:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204597.916162:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204597.916165:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204597.916167:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204597.916168:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204597.916175:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:2.0:1788204597.916176:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204597.916179:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204597.916181:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204597.916183:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204597.916185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204597.916188:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204597.916189:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204597.916195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204597.916198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204597.916201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204597.916204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204597.916205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204597.916207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204597.916210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204597.916212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204598.108685:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204598.108747:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204598.109315:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204598.109318:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204598.109652:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204598.109654:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204598.109891:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204598.109893:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204598.110046:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204598.110048:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1788204598.940148:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204598.940155:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204598.940158:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204598.940158:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204598.940162:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204598.940162:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204598.940166:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204598.940167:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204598.940169:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204598.940169:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204598.940173:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204598.940175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204598.940176:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204598.940178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204598.940182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204598.940182:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204598.940185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204599.452186:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320958213, at 4320958272 00000100:00000010:2.0:1788204599.452198:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc420. 00010000:00000001:0.0:1788204599.964168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204599.964170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204599.964174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204599.964178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204599.964179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204599.964181:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204599.964182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204599.964188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204599.964189:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204599.964190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204599.964196:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204599.964203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204599.964204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204599.964207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204599.964210:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204599.964216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204599.964222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204599.964225:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204599.964229:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204599.964231:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204600.988183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204600.988186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204600.988190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204600.988198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204600.988198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204600.988201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204600.988203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204600.988206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204600.988206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204600.988209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204600.988210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204600.988213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204600.988213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204600.988216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204600.988216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204600.988220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204600.988222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012160:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204602.012162:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204602.012166:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204602.012168:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204602.012172:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204602.012176:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204602.012177:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204602.012181:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204602.012182:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204602.012185:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204602.012186:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204602.012189:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204602.012192:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204602.012192:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204602.012197:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204602.012198:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204602.012202:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012203:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204602.012210:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204602.012212:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204602.012214:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012215:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204602.012216:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012217:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012219:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204602.012223:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204602.012224:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204602.012225:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012226:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204602.012226:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012227:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204602.012228:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204603.036179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204603.036184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204603.036188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204603.036189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204603.036193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204603.036196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204603.036198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204603.036200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204603.036204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204603.036205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204603.036209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204603.036210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204603.036214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204603.036217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204603.036217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204603.036223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204603.036230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204603.036240:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204603.036244:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204603.036247:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204603.228658:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204603.228667:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204603.229303:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204603.229305:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204603.229492:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204603.229494:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204603.229668:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204603.229670:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204603.229885:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204603.229887:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000100:00000001:2.0:1788204604.060175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204604.060178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204604.060182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204604.060188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204604.060189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204604.060193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204604.060193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204604.060199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204604.060200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204604.060203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204604.060206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204604.060207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204604.060210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204604.060211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204604.060216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204604.060218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204604.060224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204604.572195:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320963333, at 4320963392 00000100:00000010:2.0:1788204604.572205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fca80. 00000100:00000001:2.0:1788204605.084191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204605.084196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204605.084200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204605.084201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204605.084205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204605.084210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204605.084213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204605.084214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204605.084220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204605.084222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204605.084224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204605.084227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204605.084227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204605.084230:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204605.084235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204605.084235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204605.084241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.108185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204606.108192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204606.108194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204606.108198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.108199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204606.108207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204606.108208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204606.108213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.108214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204606.108220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204606.108221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204606.108223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204606.108225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204606.108227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204606.108231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204606.108232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204606.108236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204606.108260:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204606.108263:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204606.108266:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204607.132185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204607.132196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204607.132198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204607.132200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204607.132201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204607.132206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204607.132206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204607.132210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204607.132213:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204607.132214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204607.132216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204607.132217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204607.132219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204607.132221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204607.132223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204607.132224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204607.132228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204608.156196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204608.156202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204608.156204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204608.156208:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204608.156209:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204608.156213:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204608.156217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204608.156218:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204608.156224:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204608.156228:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204608.156233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204608.156237:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204608.156238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204608.156243:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204608.156244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204608.156248:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204608.156250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204608.156252:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204608.156254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204608.156260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204608.156262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204608.156265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204608.156266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204608.348786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204608.348790:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204608.349487:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204608.349489:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204608.349736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204608.349738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204608.349968:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204608.349970:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204608.350301:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204608.350303:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00010000:00000001:0.0:1788204609.180165:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204609.180169:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204609.180173:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204609.180174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204609.180178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204609.180179:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204609.180181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204609.180182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204609.180184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204609.180186:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204609.180190:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204609.180191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204609.180194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204609.180195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204609.180199:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204609.180202:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204609.180204:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204609.180216:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204609.180219:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204609.180221:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204609.692190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320968453, at 4320968512 00000100:00000010:2.0:1788204609.692206:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fca20. 00000100:00000001:2.0:1788204610.204192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204610.204194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204610.204199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204610.204201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204610.204204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204610.204205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204610.204207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204610.204212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204610.204213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204610.204214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204610.204219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204610.204221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204610.204221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204610.204227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204610.204233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204610.204238:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204610.204251:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.228165:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204611.228169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204611.228172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.228175:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204611.228178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204611.228178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204611.228181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204611.228183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204611.228185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204611.228187:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204611.228188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.228191:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204611.228194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204611.228195:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204611.228198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204611.228199:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204611.228203:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204612.252184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204612.252189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204612.252191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204612.252194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204612.252200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204612.252203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204612.252205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204612.252208:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204612.252210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204612.252212:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204612.252215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204612.252216:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204612.252220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204612.252223:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204612.252226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252227:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252232:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252236:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204612.252243:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204612.252245:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204612.252247:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252249:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204612.252251:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252253:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204612.252262:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204612.252263:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204612.252265:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252266:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204612.252268:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252269:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204612.252271:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204612.252308:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204612.252311:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204612.252314:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204613.276165:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204613.276172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204613.276174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204613.276177:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204613.276178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204613.276184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204613.276185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204613.276187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204613.276192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204613.276192:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204613.276196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204613.276197:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204613.276199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204613.276201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204613.276203:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204613.276207:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204613.276211:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204613.468193:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204613.468203:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204613.468206:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204613.468209:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204613.468213:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204613.468220:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26299 00000400:00000200:0.0:1788204613.468230:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204613.468236:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204613.468240:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204613.468243:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468246:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.468248:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468251:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204613.468254:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204613.468256:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204613.468259:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204613.468263:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204613.468265:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204613.468268:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204613.468271:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468273:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.468274:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468276:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204613.468279:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204613.468282:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468284:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204613.468287:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204613.468291:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204613.468293:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468296:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204613.468298:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204613.468307:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204613.468311:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204613.468314:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204613.468315:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468318:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204613.468321:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204613.468325:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204613.468331:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204613.468337:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204613.468346:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543449984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204613.468356:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204613.468360:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204613.468364:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543449984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204613.468384:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204613.468387:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204613.468391:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204613.468392:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204613.468393:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204613.468396:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204613.468398:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26299 00000400:00000200:0.0:1788204613.468401:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204613.468405:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204613.468407:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204613.468410:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468412:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.468414:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468416:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204613.468419:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204613.468421:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204613.468424:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204613.468427:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204613.468430:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204613.468431:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204613.468436:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468440:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.468444:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468446:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204613.468450:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204613.468456:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468472:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204613.468476:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204613.468482:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204613.468489:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468494:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204613.468500:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204613.468505:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468509:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204613.468515:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204613.468518:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204613.468521:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204613.468522:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468528:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204613.468529:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204613.468531:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204613.468538:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204613.468544:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204613.468553:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543450112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204613.468563:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204613.468566:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204613.468571:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543450112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204613.468588:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204613.468593:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204613.468601:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204613.468604:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204613.468607:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204613.468612:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204613.468618:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26299 00000400:00000200:0.0:1788204613.468631:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204613.468635:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204613.468639:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204613.468646:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468650:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.468655:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468661:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204613.468667:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204613.468671:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204613.468676:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204613.468680:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204613.468685:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204613.468689:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204613.468694:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468698:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.468702:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468705:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204613.468709:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204613.468712:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468716:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204613.468719:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204613.468722:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204613.468725:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204613.468726:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204613.468727:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204613.468731:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000800:00000200:3.1:1788204613.468732:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000008:00000001:0.0:1788204613.468735:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468740:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204613.468745:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204613.468755:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204613.468758:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204613.468762:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468766:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204613.468767:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204613.468769:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204613.468774:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204613.468779:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204613.468787:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543450240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204613.468819:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204613.468823:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204613.468827:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543450240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204613.468832:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204613.468839:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204613.468843:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204613.468846:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204613.468848:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204613.468852:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204613.468854:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26299 00000400:00000200:0.0:1788204613.468860:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204613.468863:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204613.468866:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204613.468869:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468873:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.468875:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468879:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204613.468881:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204613.468883:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204613.468886:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204613.468888:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204613.468889:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204613.468894:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204613.468897:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468899:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.468901:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468905:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204613.468907:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204613.468911:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204613.468915:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204613.468918:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204613.468921:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204613.468925:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468928:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204613.468931:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204613.468934:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468938:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204613.468944:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204613.468949:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204613.468954:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204613.468958:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.468962:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204613.468964:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204613.468966:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204613.468974:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204613.468980:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204613.468990:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543450368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204613.469009:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204613.469014:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204613.469022:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543450368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204613.469043:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204613.469046:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204613.469053:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204613.469057:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204613.469059:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204613.469064:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204613.469068:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26299 00000400:00000200:0.0:1788204613.469076:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204613.469081:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204613.469084:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204613.469110:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.469113:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.469114:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.469116:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204613.469118:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204613.469118:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204613.469120:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204613.469121:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204613.469123:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204613.469124:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204613.469126:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.469130:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204613.469134:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204613.469136:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204613.469141:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204613.469148:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204613.469153:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204613.469158:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204613.469162:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204613.469167:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.469173:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204613.469175:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204613.469180:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.469182:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204613.469187:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204613.469191:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204613.469195:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204613.469199:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204613.469204:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204613.469207:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204613.469210:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204613.469217:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204613.469222:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204613.469228:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543450496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204613.469243:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204613.469247:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204613.469255:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543450496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204613.469270:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204613.469275:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26309) 00000800:00000200:3.1:1788204613.469360:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204613.469361:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1788204613.469571:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204613.469573:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1788204613.469823:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204613.469826:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:3.1:1788204613.470147:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204613.470149:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000100:00000001:2.0:1788204614.300183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204614.300186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204614.300190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204614.300191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204614.300195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204614.300197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204614.300200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204614.300203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204614.300204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204614.300206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204614.300210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204614.300211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204614.300214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204614.300216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204614.300221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204614.300226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204614.300230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204614.812199:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320973573, at 4320973632 00000100:00000010:2.0:1788204614.812212:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcba0. 00000100:00000001:2.0:1788204615.324203:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204615.324208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204615.324211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204615.324214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204615.324214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204615.324224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204615.324224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204615.324227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204615.324231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204615.324234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204615.324236:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204615.324237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204615.324243:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204615.324245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204615.324247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204615.324248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204615.324261:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204615.324276:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204615.324280:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204615.324284:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204616.348180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204616.348185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204616.348188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204616.348189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.348191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204616.348197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204616.348197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204616.348201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.348202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204616.348205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204616.348208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204616.348209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204616.348211:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204616.348213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204616.348215:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204616.348219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204616.348222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204617.372181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204617.372189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204617.372189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204617.372194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204617.372194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204617.372200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204617.372204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204617.372204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204617.372208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204617.372210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204617.372215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204617.372216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204617.372221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204617.372221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204617.372228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204617.372229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204617.372232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204618.396196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204618.396202:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204618.396205:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204618.396208:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204618.396212:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204618.396214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204618.396219:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204618.396221:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204618.396226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204618.396228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204618.396231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204618.396233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204618.396235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204618.396237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204618.396254:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204618.396258:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204618.396260:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204618.396331:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204618.396335:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204618.396336:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204618.396339:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204618.396341:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204618.396343:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204618.396345:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204618.396347:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204618.396349:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204618.588161:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204618.588169:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204618.588171:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204618.588174:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204618.588177:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204618.588182:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26304 00000400:00000200:0.0:1788204618.588192:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204618.588199:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204618.588202:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204618.588206:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588209:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588211:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588213:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204618.588217:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204618.588218:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204618.588221:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204618.588224:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204618.588226:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204618.588228:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204618.588231:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588233:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588234:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588236:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204618.588238:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204618.588242:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588244:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204618.588247:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204618.588250:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204618.588253:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588255:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204618.588257:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204618.588266:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204618.588269:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204618.588272:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204618.588274:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588276:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204618.588279:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204618.588282:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204618.588291:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204618.588297:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204618.588306:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543450624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204618.588315:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204618.588319:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204618.588323:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543450624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204618.588346:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204618.588349:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204618.588352:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204618.588353:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204618.588355:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204618.588357:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204618.588360:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26304 00000400:00000200:0.0:1788204618.588364:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204618.588366:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204618.588369:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204618.588384:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588386:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588388:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588389:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204618.588391:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204618.588392:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204618.588394:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204618.588395:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204618.588399:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204618.588401:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204618.588405:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588407:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588410:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588412:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204618.588415:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204618.588418:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588422:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204618.588425:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204618.588432:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204618.588435:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588439:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204618.588442:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204618.588445:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588449:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204618.588453:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204618.588458:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204618.588461:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204618.588463:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588466:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204618.588468:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204618.588471:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204618.588476:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204618.588479:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204618.588485:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543450752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204618.588492:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204618.588495:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204618.588500:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543450752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204618.588510:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204618.588513:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204618.588518:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204618.588520:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204618.588521:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204618.588523:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204618.588525:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26304 00000400:00000200:0.0:1788204618.588540:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204618.588544:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204618.588546:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204618.588550:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588555:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588558:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588561:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204618.588564:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204618.588566:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204618.588569:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204618.588570:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204618.588574:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204618.588576:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204618.588580:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588583:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588585:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588589:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204618.588590:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204618.588593:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588596:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204618.588597:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204618.588599:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204618.588602:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588604:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204618.588607:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204618.588608:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588610:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204618.588612:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204618.588615:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204618.588618:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204618.588619:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588620:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204618.588621:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204618.588624:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204618.588628:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204618.588632:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204618.588637:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543450880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204618.588647:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204618.588651:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204618.588657:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543450880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204618.588665:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204618.588669:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204618.588675:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204618.588677:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204618.588678:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204618.588680:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204618.588683:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26304 00000400:00000200:0.0:1788204618.588689:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204618.588695:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204618.588700:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204618.588704:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588707:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588711:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588713:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204618.588714:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204618.588716:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204618.588719:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204618.588720:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204618.588722:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204618.588724:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204618.588728:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588729:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588730:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588731:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204618.588732:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204618.588734:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588735:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204618.588736:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204618.588737:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204618.588738:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588739:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204618.588740:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204618.588742:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588742:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204618.588744:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204618.588745:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204618.588747:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204618.588747:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588748:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204618.588749:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204618.588750:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204618.588752:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204618.588753:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204618.588755:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543451008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204618.588759:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204618.588761:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204618.588763:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543451008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204618.588767:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204618.588769:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204618.588770:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204618.588772:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204618.588775:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204618.588777:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204618.588781:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26304 00000400:00000200:0.0:1788204618.588788:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.1:1788204618.588789:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:0.0:1788204618.588791:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204618.588795:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1788204618.588796:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000040:0.0:1788204618.588798:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588802:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588806:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588808:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204618.588810:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204618.588812:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204618.588814:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204618.588816:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204618.588817:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204618.588818:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204618.588821:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588824:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204618.588827:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588830:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204618.588833:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204618.588837:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204618.588839:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204618.588841:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204618.588844:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204618.588846:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588848:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204618.588849:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204618.588852:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588853:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204618.588856:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204618.588860:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204618.588863:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204618.588865:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204618.588868:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204618.588870:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204618.588873:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204618.588876:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204618.588880:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204618.588885:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543451136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204618.588908:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204618.588910:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204618.588914:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543451136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204618.588920:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204618.588924:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26315) 00000800:00000200:3.1:1788204618.589251:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204618.589253:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204618.589542:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204618.589545:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204618.589803:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204618.589804:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788204618.589959:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204618.589961:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788204619.420178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204619.420184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204619.420185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204619.420189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204619.420189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204619.420196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204619.420198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204619.420202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204619.420202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204619.420207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204619.420209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204619.420212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204619.420214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204619.420216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204619.420221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204619.420221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204619.420224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204619.932173:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320978693, at 4320978752 00000100:00000010:2.0:1788204619.932191:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcde0. 00000100:00000001:2.0:1788204620.444191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204620.444196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204620.444200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204620.444201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204620.444204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204620.444210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204620.444212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204620.444217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204620.444217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204620.444224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204620.444227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204620.444231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204620.444235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204620.444236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204620.444239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204620.444239:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204620.444245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204621.468166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204621.468171:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204621.468174:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204621.468176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204621.468178:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204621.468180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204621.468181:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204621.468185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204621.468187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204621.468187:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204621.468190:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204621.468191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204621.468195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204621.468196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204621.468200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204621.468202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204621.468203:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204621.468223:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204621.468225:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204621.468227:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204622.492201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204622.492205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204622.492211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.492211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204622.492215:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204622.492221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204622.492223:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204622.492227:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204622.492228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204622.492232:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.492234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204622.492236:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204622.492240:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.492242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204622.492244:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204622.492247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204622.492248:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204622.492257:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204622.492260:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204622.492262:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492263:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492267:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204622.492269:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492271:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492273:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204622.492278:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204622.492280:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204622.492283:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492285:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204622.492286:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492288:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204622.492290:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204623.516181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204623.516182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204623.516187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.516189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204623.516192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204623.516195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204623.516195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204623.516199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.516199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204623.516203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204623.516204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204623.516207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.516209:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204623.516211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204623.516213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.516214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204623.516220:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204623.708201:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204623.708209:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204623.708211:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204623.708214:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204623.708217:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204623.708223:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26310 00000400:00000200:2.0:1788204623.708232:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204623.708237:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204623.708239:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204623.708242:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708244:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708246:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708247:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204623.708251:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204623.708253:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204623.708255:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204623.708257:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204623.708259:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204623.708260:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204623.708262:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708263:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708264:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708265:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204623.708267:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204623.708269:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708272:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204623.708275:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204623.708278:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204623.708281:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708283:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204623.708308:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204623.708317:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204623.708321:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204623.708325:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204623.708326:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708328:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204623.708331:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204623.708334:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204623.708342:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204623.708347:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204623.708355:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543451264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204623.708361:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204623.708365:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204623.708368:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543451264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204623.708383:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204623.708385:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204623.708387:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204623.708388:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204623.708390:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204623.708391:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204623.708393:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26310 00000400:00000200:2.0:1788204623.708396:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204623.708398:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204623.708400:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204623.708402:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708403:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708404:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708406:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204623.708407:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204623.708408:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204623.708410:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204623.708411:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204623.708413:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204623.708414:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204623.708416:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708418:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708419:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708421:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204623.708422:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204623.708424:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708426:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204623.708430:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204623.708435:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204623.708438:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708441:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204623.708444:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204623.708447:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708448:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204623.708453:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204623.708461:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204623.708464:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204623.708467:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708469:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204623.708470:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204623.708471:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204623.708473:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204623.708476:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204623.708483:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543451392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204623.708494:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204623.708497:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204623.708504:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543451392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204623.708511:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204623.708514:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204623.708519:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204623.708520:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204623.708523:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204623.708525:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204623.708527:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26310 00000400:00000200:2.0:1788204623.708530:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204623.708534:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204623.708538:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204623.708542:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708544:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708548:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708550:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204623.708551:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204623.708554:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204623.708556:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204623.708557:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204623.708558:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204623.708560:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204623.708561:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708563:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708566:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708568:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204623.708569:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204623.708572:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708574:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204623.708578:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204623.708583:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204623.708587:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708591:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204623.708594:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204623.708600:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708604:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204623.708610:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204623.708616:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204623.708620:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204623.708621:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708625:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204623.708627:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204623.708629:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204623.708635:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204623.708639:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204623.708645:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543451520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204623.708653:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204623.708657:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204623.708660:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543451520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204623.708671:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204623.708674:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204623.708679:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204623.708680:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204623.708683:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204623.708687:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204623.708691:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26310 00000400:00000200:2.0:1788204623.708725:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204623.708737:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204623.708741:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204623.708748:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708752:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708756:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708758:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204623.708762:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:3.1:1788204623.708763:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:2.0:1788204623.708764:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204623.708767:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204623.708771:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000800:00000200:3.1:1788204623.708771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1788204623.708774:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204623.708777:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204623.708781:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708784:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708787:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708792:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204623.708794:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204623.708796:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708798:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204623.708800:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204623.708803:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204623.708806:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708810:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204623.708813:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204623.708815:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708818:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204623.708822:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204623.708827:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204623.708829:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204623.708830:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708834:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204623.708836:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204623.708838:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204623.708841:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204623.708846:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204623.708852:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543451648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204623.708859:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204623.708863:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204623.708870:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543451648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204623.708884:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204623.708887:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204623.708890:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204623.708892:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204623.708893:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204623.708895:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204623.708899:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26310 00000400:00000200:2.0:1788204623.708904:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204623.708910:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204623.708912:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204623.708917:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708918:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708920:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708922:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204623.708926:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204623.708927:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204623.708930:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204623.708933:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204623.708937:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204623.708940:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204623.708947:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708952:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204623.708956:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708960:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204623.708966:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204623.708971:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204623.708976:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204623.708979:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204623.708983:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204623.708988:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.708993:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204623.708996:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204623.709001:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.709005:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204623.709012:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204623.709017:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204623.709023:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204623.709026:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204623.709031:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204623.709034:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204623.709037:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204623.709044:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204623.709050:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204623.709059:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543451776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204623.709071:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204623.709074:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204623.709082:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543451776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204623.709118:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204623.709125:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26320) 00000800:00000200:3.1:1788204623.709270:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204623.709272:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204623.709456:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204623.709458:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204623.709612:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204623.709613:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204623.709796:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204623.709798:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204624.540181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204624.540182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204624.540185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204624.540188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204624.540190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204624.540191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204624.540194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204624.540196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204624.540199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204624.540200:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204624.540202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204624.540203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204624.540208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204624.540209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204624.540213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204624.540213:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204624.540217:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204624.540235:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204624.540238:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204624.540240:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204625.052181:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320983813, at 4320983872 00000100:00000010:2.0:1788204625.052194:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc5a0. 00000100:00000001:2.0:1788204625.564166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204625.564173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204625.564173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204625.564176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204625.564178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204625.564183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204625.564185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204625.564188:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204625.564189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204625.564192:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204625.564195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204625.564196:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204625.564198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204625.564200:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204625.564204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204625.564206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204625.564207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204626.588161:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204626.588167:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204626.588168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204626.588171:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204626.588172:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204626.588176:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204626.588179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204626.588180:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204626.588182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204626.588184:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204626.588187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204626.588188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204626.588191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204626.588193:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204626.588196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204626.588198:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204626.588200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.612169:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204627.612173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204627.612176:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204627.612178:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204627.612180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204627.612181:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204627.612186:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204627.612187:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.612193:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204627.612194:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204627.612197:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.612197:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204627.612202:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.612206:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204627.612207:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204627.612209:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204627.612211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204627.612215:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204627.612225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204627.612233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204627.612234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204627.612237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204627.612238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204627.612255:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204627.612256:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204627.612258:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204628.636166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204628.636173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204628.636175:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204628.636180:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204628.636184:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204628.636187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204628.636187:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204628.636189:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204628.636192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204628.636193:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204628.636212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204628.636215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204628.636219:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204628.636221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204628.636225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204628.636231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204628.636234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204628.828740:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204628.828745:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204628.829229:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204628.829230:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204628.829487:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204628.829489:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204628.829759:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204628.829761:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204628.830020:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204628.830022:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788204629.660172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204629.660174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204629.660178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204629.660179:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204629.660193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204629.660196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204629.660197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204629.660199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204629.660204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204629.660204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204629.660208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204629.660210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204629.660211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204629.660214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204629.660216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204629.660220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204629.660227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204630.172161:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320988933, at 4320988992 00000100:00000010:2.0:1788204630.172169:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00000100:00000001:2.0:1788204630.684147:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204630.684153:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204630.684154:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204630.684158:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204630.684161:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204630.684162:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204630.684177:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204630.684177:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204630.684184:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204630.684185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204630.684188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204630.684188:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204630.684194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204630.684194:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204630.684197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204630.684201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204630.684204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204630.684226:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204630.684229:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204630.684231:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204631.708203:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204631.708208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204631.708213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204631.708213:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204631.708216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204631.708221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204631.708222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204631.708224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204631.708228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204631.708229:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204631.708230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204631.708235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204631.708235:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204631.708238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204631.708242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204631.708246:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204631.708250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.732204:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204632.732212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204632.732212:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204632.732214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204632.732221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204632.732224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204632.732229:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204632.732233:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204632.732234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204632.732237:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.732238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204632.732242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204632.732243:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204632.732247:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.732247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204632.732253:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204632.732253:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732259:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204632.732266:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204632.732268:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204632.732270:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732272:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204632.732273:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204632.732282:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204632.732283:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204632.732284:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732287:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204632.732288:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732289:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204632.732291:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204633.756185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204633.756191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204633.756193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204633.756195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204633.756197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204633.756200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204633.756204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204633.756205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204633.756208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204633.756209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204633.756215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204633.756216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204633.756220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204633.756220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204633.756228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204633.756229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204633.756233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204633.756256:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204633.756259:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204633.756262:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204633.948701:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204633.948709:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204633.949246:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204633.949248:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204633.949426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204633.949428:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204633.949581:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204633.949582:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204633.949766:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204633.949767:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1788204634.780209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204634.780217:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204634.780221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204634.780228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204634.780230:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204634.780240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204634.780241:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204634.780249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204634.780252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204634.780257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204634.780259:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204634.780263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204634.780264:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204634.780268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204634.780271:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204634.780275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204634.780281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204635.292223:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320994053, at 4320994112 00000100:00000010:2.0:1788204635.292246:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c00. 00000100:00000001:2.0:1788204635.804223:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204635.804235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204635.804235:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204635.804242:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204635.804244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204635.804251:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204635.804256:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204635.804257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204635.804262:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204635.804263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204635.804272:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204635.804274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204635.804281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204635.804282:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204635.804291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204635.804294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204635.804294:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204636.828182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204636.828185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204636.828188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204636.828190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204636.828193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204636.828194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204636.828195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204636.828201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204636.828201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204636.828204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204636.828206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204636.828208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204636.828211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204636.828211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204636.828216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204636.828220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204636.828223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204636.828234:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204636.828236:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204636.828238:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204637.852191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204637.852198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204637.852199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204637.852203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204637.852205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204637.852208:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204637.852211:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204637.852211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204637.852216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204637.852218:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204637.852220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204637.852222:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.852224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204637.852225:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.852227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204637.852229:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204637.852231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204637.852234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204637.852236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204637.852243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204637.852246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204637.852250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204637.852251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204638.876188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204638.876196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204638.876196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204638.876200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204638.876202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204638.876206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204638.876210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204638.876216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204638.876218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204638.876221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204638.876222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204638.876225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204638.876230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204638.876230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204638.876234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204638.876241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204638.876242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204639.068208:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204639.068217:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204639.068218:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204639.068221:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204639.068225:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204639.068230:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26325 00000400:00000200:2.0:1788204639.068238:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204639.068243:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204639.068246:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204639.068248:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068250:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068252:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068254:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204639.068259:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204639.068260:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204639.068263:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204639.068266:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204639.068269:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204639.068272:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204639.068275:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068277:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068279:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068281:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204639.068284:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204639.068287:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068290:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204639.068293:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204639.068297:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204639.068299:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068300:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204639.068302:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204639.068311:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204639.068313:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204639.068317:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204639.068318:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068320:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204639.068322:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204639.068325:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204639.068330:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204639.068336:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204639.068347:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543453184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204639.068365:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204639.068370:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204639.068375:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543453184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204639.068392:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204639.068394:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204639.068397:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204639.068398:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204639.068399:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204639.068401:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204639.068403:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26325 00000400:00000200:2.0:1788204639.068406:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204639.068408:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204639.068410:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204639.068412:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068413:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068415:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068416:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204639.068418:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204639.068419:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204639.068420:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204639.068422:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204639.068423:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204639.068425:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204639.068426:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068428:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068430:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068434:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204639.068436:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204639.068441:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068445:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204639.068446:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204639.068451:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204639.068456:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068460:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204639.068465:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204639.068469:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068472:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204639.068475:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204639.068480:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204639.068484:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204639.068487:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068491:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204639.068493:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204639.068497:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204639.068501:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204639.068505:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204639.068512:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543453312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204639.068523:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204639.068527:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204639.068531:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543453312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204639.068540:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204639.068546:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204639.068548:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204639.068550:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204639.068553:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204639.068557:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204639.068561:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26325 00000400:00000200:2.0:1788204639.068565:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204639.068572:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204639.068576:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204639.068580:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068585:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068589:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068593:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204639.068597:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204639.068598:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204639.068600:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204639.068601:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204639.068604:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204639.068607:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204639.068609:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068611:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068615:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068618:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204639.068619:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204639.068621:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068622:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204639.068623:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204639.068625:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204639.068627:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068631:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204639.068633:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204639.068637:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068640:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204639.068645:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204639.068649:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204639.068653:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204639.068655:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068659:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204639.068663:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204639.068667:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204639.068672:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204639.068678:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204639.068688:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543453440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204639.068704:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1788204639.068705:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204639.068707:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204639.068713:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543453440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204639.068718:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204639.068721:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204639.068725:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204639.068727:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204639.068728:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204639.068732:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204639.068737:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26325 00000400:00000200:2.0:1788204639.068744:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204639.068749:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204639.068751:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204639.068755:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068758:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068761:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068764:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204639.068768:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204639.068770:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204639.068773:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204639.068777:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204639.068781:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204639.068784:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204639.068789:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068791:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068792:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068793:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204639.068795:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204639.068800:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068804:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204639.068807:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204639.068811:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204639.068815:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068819:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204639.068822:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204639.068828:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068832:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204639.068838:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204639.068844:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204639.068849:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204639.068851:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.068854:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204639.068856:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204639.068859:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204639.068864:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204639.068868:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204639.068876:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543453568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204639.068884:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204639.068887:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204639.068893:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543453568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204639.068903:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204639.068907:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204639.068911:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204639.068915:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204639.068917:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204639.068920:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204639.068925:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26325 00000400:00000200:2.0:1788204639.068931:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204639.068936:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204639.068938:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204639.068943:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068947:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068952:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068957:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204639.068960:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204639.068963:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204639.068965:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204639.068969:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204639.068973:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204639.068976:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204639.068982:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068986:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204639.068990:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204639.068993:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204639.068996:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204639.069000:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204639.069004:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204639.069005:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204639.069008:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204639.069011:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.069012:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204639.069014:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204639.069017:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.069032:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204639.069036:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:2.0:1788204639.069037:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204639.069041:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204639.069044:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204639.069047:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.069049:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204639.069050:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204639.069052:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204639.069056:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204639.069061:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204639.069070:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543453696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204639.069128:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204639.069133:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204639.069142:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543453696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204639.069159:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204639.069168:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26335) 00000800:00000200:1.1:1788204639.069278:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204639.069446:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204639.069579:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204639.900185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204639.900189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204639.900191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204639.900194:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204639.900199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204639.900203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204639.900204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204639.900206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204639.900210:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204639.900211:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204639.900216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204639.900216:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204639.900220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204639.900226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204639.900230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204639.900235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204639.900239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204639.900263:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204639.900273:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204639.900276:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204640.412185:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4320999173, at 4320999232 00000100:00000010:2.0:1788204640.412199:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00000100:00000001:2.0:1788204640.924153:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204640.924160:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204640.924162:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204640.924167:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204640.924170:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204640.924173:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204640.924176:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204640.924178:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204640.924181:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204640.924193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204640.924202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204640.924211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204640.924216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204640.924224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204640.924227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204640.924231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204640.924233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204641.948172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204641.948178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204641.948180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204641.948184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204641.948189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204641.948192:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204641.948194:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204641.948196:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204641.948199:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204641.948215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204641.948221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204641.948230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204641.948233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204641.948238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204641.948239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204641.948244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204641.948246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788204642.140458:0:256120:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:0.0:1788204642.140519:0:256120:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204642.140537:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204642.140541:0:256120:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:0.0:1788204642.140548:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204642.140553:0:256120:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:0.0:1788204642.140559:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204642.140564:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:0.0:1788204642.140573:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204642.140578:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204642.140587:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6440 00000400:00000010:0.0:1788204642.140591:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6440. 00000100:00000001:0.0:1788204642.140598:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204642.140608:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d77480 x1875055543453184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140640:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204642.140645:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77480 x1875055543453184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140654:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.140657:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204642.140673:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204642.140675:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:0.0:1788204642.140679:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204642.140683:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204642.140685:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6cc0 00000400:00000010:0.0:1788204642.140687:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6cc0. 00000100:00000001:0.0:1788204642.140690:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204642.140696:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d77800 x1875055543453312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140705:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204642.140708:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543453312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140715:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.140717:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204642.140723:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204642.140726:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000400:00000200:0.0:1788204642.140729:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204642.140732:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204642.140735:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6880 00000400:00000010:0.0:1788204642.140736:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6880. 00000100:00000001:0.0:1788204642.140739:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204642.140743:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74000 x1875055543453440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140752:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204642.140755:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543453440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140761:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.140763:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204642.140769:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204642.140772:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000400:00000200:0.0:1788204642.140775:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204642.140778:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204642.140781:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:0.0:1788204642.140783:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:0.0:1788204642.140786:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204642.140791:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74700 x1875055543453568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140800:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204642.140802:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543453568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140808:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.140810:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204642.140814:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204642.140817:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:0.0:1788204642.140819:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204642.140821:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204642.140824:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6770 00000400:00000010:0.0:1788204642.140826:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6770. 00000100:00000001:0.0:1788204642.140829:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204642.140834:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75180 x1875055543453696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140842:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204642.140844:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543453696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204664 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204642.140850:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204642.140852:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204642.972186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204642.972195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204642.972199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204642.972205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204642.972211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204642.972214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204642.972216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204642.972222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204642.972227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204642.972230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204642.972232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204642.972234:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204642.972242:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204642.972242:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972245:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972250:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204642.972253:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204642.972263:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204642.972265:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204642.972268:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972270:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204642.972271:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972278:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204642.972286:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204642.972294:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204642.972296:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972298:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204642.972299:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972301:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204642.972304:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204642.972333:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204642.972336:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204642.972339:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204643.996220:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204643.996223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204643.996229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204643.996230:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204643.996238:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204643.996239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204643.996245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204643.996252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204643.996253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204643.996255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204643.996264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204643.996266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204643.996271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204643.996272:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204643.996281:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204643.996287:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204643.996294:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204644.188162:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204644.188172:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204644.188174:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204644.188178:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204644.188183:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204644.188191:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26330 00000400:00000200:2.0:1788204644.188202:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204644.188209:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204644.188212:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204644.188216:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188220:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188223:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188226:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204644.188230:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204644.188232:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204644.188236:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204644.188240:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204644.188243:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204644.188246:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204644.188250:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188252:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188255:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188258:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204644.188261:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204644.188265:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188267:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204644.188271:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204644.188274:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204644.188278:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188280:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204644.188283:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204644.188291:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204644.188294:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204644.188298:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204644.188299:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188301:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204644.188303:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204644.188307:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204644.188314:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204644.188320:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204644.188329:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543453824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204644.188337:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204644.188340:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204644.188346:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543453824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204644.188370:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204644.188374:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204644.188378:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204644.188380:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204644.188381:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204644.188385:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204644.188387:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26330 00000400:00000200:2.0:1788204644.188391:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204644.188394:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204644.188396:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204644.188399:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188401:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188403:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188405:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204644.188407:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204644.188408:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204644.188409:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204644.188412:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204644.188413:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204644.188415:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204644.188418:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188420:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188421:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188423:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204644.188425:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204644.188427:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188429:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204644.188431:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204644.188436:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204644.188438:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188441:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204644.188445:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204644.188449:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188450:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204644.188454:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204644.188456:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204644.188458:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204644.188459:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188461:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204644.188462:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204644.188463:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204644.188466:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204644.188470:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204644.188475:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543453952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204644.188481:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204644.188483:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204644.188486:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543453952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204644.188493:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204644.188496:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204644.188499:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204644.188500:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204644.188501:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204644.188504:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204644.188506:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26330 00000400:00000200:2.0:1788204644.188510:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204644.188512:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204644.188515:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204644.188517:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188519:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188520:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188523:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204644.188525:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204644.188526:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204644.188528:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204644.188529:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204644.188531:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204644.188533:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204644.188536:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188537:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188539:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188540:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204644.188542:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204644.188545:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188547:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204644.188548:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204644.188550:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204644.188552:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188554:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204644.188556:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204644.188558:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188559:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204644.188563:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204644.188565:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204644.188567:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204644.188568:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188570:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204644.188571:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204644.188573:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204644.188576:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204644.188578:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204644.188583:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543454080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204644.188594:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204644.188596:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204644.188599:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543454080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204644.188607:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204644.188610:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204644.188613:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204644.188614:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204644.188615:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204644.188618:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204644.188620:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26330 00000400:00000200:2.0:1788204644.188623:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204644.188626:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204644.188628:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204644.188630:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188632:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188634:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.1:1788204644.188634:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204644.188636:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204644.188637:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:1.1:1788204644.188637:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000020:00000040:2.0:1788204644.188638:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204644.188640:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204644.188642:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204644.188643:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204644.188645:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204644.188648:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188649:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188651:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188653:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204644.188656:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204644.188658:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188660:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204644.188661:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204644.188664:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204644.188665:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188667:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204644.188669:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204644.188671:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188673:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204644.188677:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204644.188679:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204644.188681:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204644.188682:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188684:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204644.188685:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204644.188687:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204644.188690:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204644.188693:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204644.188696:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543454208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204644.188703:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204644.188705:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204644.188707:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543454208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204644.188715:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204644.188717:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204644.188720:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204644.188721:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204644.188722:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204644.188725:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204644.188728:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26330 00000400:00000200:2.0:1788204644.188731:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204644.188734:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204644.188736:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204644.188739:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188741:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188743:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188745:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204644.188748:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204644.188749:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204644.188750:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204644.188752:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204644.188754:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204644.188756:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204644.188759:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188761:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204644.188763:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188765:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204644.188767:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204644.188770:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204644.188772:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204644.188774:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204644.188777:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204644.188779:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188781:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204644.188783:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204644.188785:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188787:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204644.188790:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204644.188792:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204644.188795:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204644.188797:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204644.188799:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204644.188801:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204644.188803:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204644.188806:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204644.188809:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204644.188813:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543454336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204644.188821:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204644.188823:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204644.188826:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543454336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204644.188836:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204644.188840:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26340) 00000800:00000200:1.1:1788204644.189060:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204644.189063:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204644.189355:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204644.189357:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204644.189598:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204644.189600:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204644.189842:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204644.189844:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788204645.020182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204645.020185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204645.020189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204645.020191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204645.020193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204645.020195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204645.020196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204645.020202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204645.020204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204645.020204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204645.020209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204645.020211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204645.020211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204645.020215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.020219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204645.020222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204645.020224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204645.532206:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321004293, at 4321004352 00000100:00000010:2.0:1788204645.532238:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647900. 00000100:00000001:2.0:1788204646.044202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204646.044208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204646.044212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204646.044217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204646.044222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204646.044224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204646.044226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204646.044231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204646.044234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204646.044239:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204646.044240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204646.044246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204646.044248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204646.044253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204646.044255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204646.044259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204646.044262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204646.044289:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204646.044293:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204646.044295:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204647.068194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204647.068201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204647.068203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204647.068208:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204647.068213:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204647.068217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204647.068221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204647.068222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204647.068230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204647.068230:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204647.068244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204647.068246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204647.068249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204647.068259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204647.068261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204647.068268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204647.068269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204648.092213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204648.092224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204648.092226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204648.092230:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204648.092233:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204648.092238:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204648.092238:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204648.092244:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204648.092245:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204648.092252:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204648.092252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204648.092258:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204648.092258:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204648.092264:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204648.092264:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204648.092272:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204648.092272:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204648.092285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204648.092288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204648.092293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204648.092296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204648.092300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204648.092302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204649.116188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204649.116188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204649.116195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204649.116198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204649.116201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204649.116202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204649.116203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204649.116209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204649.116210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204649.116213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204649.116216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204649.116222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204649.116227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204649.116227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204649.116232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204649.116234:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204649.116243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204649.116255:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204649.116261:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204649.116265:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204649.308884:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204649.308892:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204649.309811:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204649.309816:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204649.310199:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204649.310201:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204649.310409:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204649.310410:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204649.310568:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204649.310570:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788204650.140142:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204650.140149:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204650.140150:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204650.140155:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204650.140159:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204650.140162:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204650.140164:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204650.140166:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204650.140170:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204650.140184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204650.140190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204650.140196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204650.140199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204650.140203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204650.140205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204650.140210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204650.140211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204650.652216:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321009413, at 4321009472 00000100:00000010:2.0:1788204650.652243:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00000100:00000001:2.0:1788204651.164208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204651.164216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204651.164219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204651.164224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204651.164228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204651.164232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204651.164234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204651.164237:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204651.164245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204651.164248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204651.164249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204651.164251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204651.164268:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204651.164269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204651.164278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204651.164285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204651.164289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.188206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204652.188214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204652.188218:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204652.188221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.188221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204652.188231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204652.188237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204652.188237:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204652.188247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.188252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204652.188254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204652.188259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.188269:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204652.188270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204652.188273:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204652.188277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204652.188279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204652.188310:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204652.188312:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204652.188314:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204653.212238:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204653.212241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204653.212249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204653.212252:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204653.212261:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204653.212271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204653.212282:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204653.212284:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204653.212291:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204653.212293:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204653.212299:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204653.212303:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204653.212309:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204653.212310:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204653.212318:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204653.212323:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212325:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212334:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212340:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204653.212344:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204653.212346:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204653.212349:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212351:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204653.212352:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212356:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212360:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212366:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204653.212369:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204653.212370:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204653.212372:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212374:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204653.212376:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212377:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204653.212379:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204654.236213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204654.236220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204654.236223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204654.236223:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204654.236231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204654.236232:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204654.236246:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204654.236248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204654.236255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204654.236256:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204654.236269:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204654.236273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204654.236273:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204654.236279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204654.236282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204654.236287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204654.236291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204654.428698:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204654.428706:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204654.429399:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204654.429401:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204654.429588:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204654.429590:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204654.429780:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204654.429782:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204654.429987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204654.429989:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788204655.260193:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204655.260199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204655.260201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204655.260204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204655.260205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204655.260212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204655.260213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204655.260216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204655.260217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204655.260221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204655.260222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204655.260223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204655.260226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204655.260228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204655.260230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204655.260238:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204655.260244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204655.772196:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321014533, at 4321014592 00000100:00000010:2.0:1788204655.772211:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00000100:00000001:2.0:1788204656.284190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204656.284193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204656.284198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204656.284201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204656.284203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204656.284206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204656.284206:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204656.284211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204656.284213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204656.284215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204656.284216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204656.284218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204656.284222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204656.284227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204656.284230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204656.284234:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204656.284237:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.308200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204657.308209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204657.308211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204657.308213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204657.308217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204657.308224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204657.308226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204657.308231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.308233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204657.308237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204657.308241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.308241:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204657.308249:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204657.308250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204657.308255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204657.308255:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204657.308270:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204658.332168:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204658.332172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204658.332176:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204658.332178:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204658.332181:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204658.332182:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204658.332187:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204658.332188:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204658.332195:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204658.332196:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204658.332202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204658.332203:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204658.332211:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204658.332211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204658.332216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204658.332217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204658.332223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204658.332228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204658.332231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204658.332236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204658.332237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204658.332242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204658.332244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.356206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204659.356211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204659.356215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204659.356216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.356220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204659.356223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204659.356226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204659.356228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204659.356229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204659.356231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204659.356236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204659.356237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.356237:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204659.356243:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204659.356248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204659.356251:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204659.356254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204659.548219:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204659.548228:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204659.548231:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204659.548234:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204659.548238:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204659.548256:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26345 00000400:00000200:2.0:1788204659.548267:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204659.548275:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204659.548279:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204659.548282:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548286:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.548288:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548291:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204659.548295:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204659.548298:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204659.548301:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204659.548305:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204659.548308:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204659.548311:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204659.548315:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548317:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.548319:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548322:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204659.548325:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204659.548327:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548329:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204659.548340:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204659.548342:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204659.548345:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.548347:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204659.548349:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204659.548358:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204659.548361:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204659.548364:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204659.548366:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.548368:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204659.548370:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204659.548374:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204659.548380:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204659.548386:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204659.548394:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543455744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204659.548408:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204659.548411:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204659.548415:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543455744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204659.548439:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204659.548443:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204659.548447:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204659.548449:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204659.548451:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204659.548455:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204659.548458:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26345 00000400:00000200:2.0:1788204659.548464:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204659.548468:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204659.548471:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204659.548475:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548478:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.548481:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548483:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204659.548487:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204659.548491:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204659.548494:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204659.548498:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204659.548503:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204659.548508:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204659.548516:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548521:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.548525:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548530:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204659.548532:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204659.548537:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548540:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204659.548541:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204659.548546:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204659.548549:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.548552:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204659.548559:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204659.548565:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.548571:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204659.548579:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204659.548585:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204659.548589:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204659.548594:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.548598:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204659.548602:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204659.548606:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204659.548615:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204659.548621:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204659.548629:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543455872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204659.548636:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204659.548640:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204659.548643:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543455872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204659.548656:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204659.548661:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204659.548685:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204659.548689:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204659.548692:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204659.548697:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204659.548704:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26345 00000400:00000200:2.0:1788204659.548712:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204659.548716:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204659.548721:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204659.548725:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548729:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.548732:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548733:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204659.548735:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204659.548737:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204659.548740:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204659.548743:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204659.548746:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204659.548748:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204659.548751:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548753:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.548754:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548755:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204659.548757:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204659.548759:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548761:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204659.548766:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204659.548770:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204659.548775:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.548779:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204659.548784:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204659.548789:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.548794:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204659.548799:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204659.548805:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204659.548812:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204659.548814:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.548815:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204659.548817:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204659.548819:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204659.548823:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204659.548830:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204659.548842:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543456000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204659.548855:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204659.548861:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204659.548869:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543456000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204659.548883:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204659.548896:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204659.548903:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204659.548907:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1788204659.548907:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204659.548910:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1788204659.548914:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1788204659.548916:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204659.548923:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26345 00000400:00000200:2.0:1788204659.548939:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204659.548948:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204659.548950:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204659.548954:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548957:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.548963:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.548966:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204659.548971:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204659.548972:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204659.548975:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204659.548979:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204659.548984:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204659.548997:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204659.549006:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.549012:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.549018:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.549022:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204659.549028:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204659.549034:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204659.549041:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204659.549046:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204659.549051:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204659.549057:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.549059:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204659.549062:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204659.549065:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.549069:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204659.549073:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204659.549077:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204659.549082:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204659.549083:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.549085:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204659.549088:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204659.549113:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204659.549118:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204659.549123:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204659.549129:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543456128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204659.549142:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204659.549145:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204659.549154:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543456128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204659.549170:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204659.549175:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204659.549181:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204659.549184:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204659.549187:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204659.549193:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204659.549197:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26345 00000400:00000200:2.0:1788204659.549202:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204659.549208:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204659.549212:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204659.549218:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.549222:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.549224:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.549226:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204659.549227:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204659.549228:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204659.549230:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204659.549232:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204659.549235:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204659.549239:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204659.549244:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.549248:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204659.549251:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204659.549256:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204659.549259:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204659.549265:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204659.549270:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204659.549272:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204659.549275:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204659.549278:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.549280:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204659.549283:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204659.549288:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.549292:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204659.549297:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204659.549301:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204659.549304:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204659.549307:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204659.549309:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204659.549311:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204659.549313:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204659.549318:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204659.549324:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204659.549331:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543456256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204659.549350:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204659.549354:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204659.549360:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543456256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204659.549365:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204659.549369:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26356) 00000800:00000200:1.1:1788204659.549659:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204659.549663:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204659.550028:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204659.550031:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204659.550388:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204659.550390:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204659.550730:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204659.550733:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1788204660.380161:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204660.380170:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204660.380176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204660.380184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204660.380185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204660.380192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204660.380192:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204660.380199:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204660.380202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204660.380203:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204660.380206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204660.380208:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204660.380215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204660.380215:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204660.380219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204660.380226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204660.380228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204660.892184:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321019653, at 4321019712 00000100:00000010:2.0:1788204660.892198:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647660. 00010000:00000001:0.0:1788204661.404195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204661.404195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204661.404201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204661.404203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204661.404208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204661.404209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204661.404213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204661.404215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204661.404217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204661.404220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204661.404222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204661.404223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204661.404226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204661.404228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204661.404232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204661.404236:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204661.404240:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204662.428196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204662.428198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204662.428204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204662.428205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204662.428207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204662.428214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204662.428217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204662.428221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204662.428222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204662.428227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204662.428230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204662.428236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204662.428236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204662.428243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204662.428244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204662.428249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204662.428250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.452183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204663.452186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204663.452190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.452192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204663.452197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204663.452203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204663.452205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204663.452209:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204663.452211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204663.452213:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.452218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204663.452219:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204663.452222:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.452225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204663.452226:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.452229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204663.452232:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204663.452235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204663.452241:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204663.452242:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204663.452244:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452245:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204663.452247:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452248:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204663.452255:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204663.452256:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204663.452258:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452259:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204663.452260:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452261:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204663.452263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.476198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204664.476203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204664.476206:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204664.476207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.476209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204664.476215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204664.476217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204664.476219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204664.476221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.476223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204664.476227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204664.476228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204664.476231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204664.476232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.476234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204664.476241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204664.476243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204664.668190:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204664.668198:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204664.668200:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204664.668203:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204664.668206:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204664.668211:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26351 00000400:00000200:2.0:1788204664.668220:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204664.668225:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204664.668227:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204664.668229:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668232:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668233:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668235:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204664.668238:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204664.668239:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204664.668241:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204664.668244:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204664.668246:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204664.668248:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204664.668251:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668252:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668254:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668256:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204664.668258:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204664.668261:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668264:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204664.668266:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204664.668270:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204664.668272:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668275:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204664.668278:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204664.668286:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204664.668290:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204664.668293:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204664.668295:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668298:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204664.668300:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204664.668303:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204664.668310:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204664.668316:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204664.668323:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543456384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204664.668330:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204664.668340:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204664.668343:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543456384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204664.668359:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204664.668361:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204664.668364:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204664.668365:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204664.668366:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204664.668368:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204664.668370:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26351 00000400:00000200:2.0:1788204664.668373:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204664.668375:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204664.668376:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204664.668378:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668380:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668381:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668383:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204664.668384:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204664.668386:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204664.668387:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204664.668388:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204664.668390:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204664.668391:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204664.668393:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668394:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668395:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668396:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204664.668398:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204664.668400:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668405:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204664.668407:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204664.668413:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204664.668416:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668417:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204664.668422:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204664.668426:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668429:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204664.668433:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204664.668436:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204664.668439:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204664.668441:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668443:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204664.668446:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204664.668448:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204664.668454:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204664.668457:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204664.668463:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543456512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204664.668473:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204664.668476:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204664.668481:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543456512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204664.668490:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204664.668492:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204664.668497:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204664.668499:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204664.668502:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204664.668504:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204664.668507:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26351 00000400:00000200:2.0:1788204664.668509:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204664.668511:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204664.668515:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204664.668519:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668521:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668522:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668527:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204664.668530:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204664.668532:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204664.668533:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204664.668535:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204664.668538:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204664.668541:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204664.668542:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668543:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668544:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668546:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204664.668550:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204664.668552:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668554:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204664.668556:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204664.668559:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204664.668562:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668567:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204664.668569:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204664.668572:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668573:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204664.668597:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204664.668601:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204664.668603:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204664.668606:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668608:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204664.668610:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204664.668613:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204664.668617:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204664.668622:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204664.668626:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543456640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204664.668642:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204664.668644:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204664.668648:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543456640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204664.668654:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204664.668657:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204664.668660:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204664.668662:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204664.668666:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204664.668668:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204664.668671:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26351 00000400:00000200:2.0:1788204664.668678:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204664.668683:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204664.668686:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204664.668691:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668695:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668701:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668706:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204664.668710:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204664.668713:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204664.668716:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204664.668720:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204664.668724:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204664.668726:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204664.668733:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668736:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668741:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668744:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204664.668750:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204664.668756:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668759:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204664.668763:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204664.668766:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204664.668771:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668777:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204664.668779:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204664.668785:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668787:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204664.668790:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:2.0:1788204664.668793:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000800:00000200:1.1:1788204664.668796:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000020:00000040:2.0:1788204664.668797:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204664.668803:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204664.668806:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668808:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204664.668812:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204664.668815:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204664.668821:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204664.668826:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204664.668833:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543456768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204664.668845:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204664.668848:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204664.668856:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543456768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204664.668878:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204664.668881:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204664.668887:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204664.668890:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204664.668892:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204664.668897:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204664.668909:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26351 00000400:00000200:2.0:1788204664.668915:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204664.668921:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204664.668924:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204664.668926:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668931:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668932:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668934:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204664.668936:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204664.668938:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204664.668940:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204664.668941:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204664.668943:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204664.668945:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204664.668949:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668950:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204664.668952:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668954:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204664.668957:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204664.668960:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204664.668962:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204664.668964:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204664.668967:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204664.668969:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668973:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204664.668974:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204664.668976:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668981:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204664.668984:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204664.668987:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204664.668991:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204664.668993:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204664.668996:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204664.668997:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204664.669000:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204664.669004:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204664.669008:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204664.669014:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543456896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204664.669037:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204664.669040:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204664.669044:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543456896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204664.669053:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204664.669058:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26361) 00000800:00000200:1.1:1788204664.669371:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204664.669374:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204664.669661:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204664.669664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204664.669941:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204664.669944:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204664.670236:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204664.670238:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00000001:2.0:1788204665.500187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204665.500194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204665.500197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204665.500199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204665.500204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204665.500205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204665.500211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204665.500214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204665.500215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204665.500217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204665.500221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204665.500225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204665.500225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204665.500231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204665.500232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204665.500240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204665.500242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204666.012187:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321024773, at 4321024832 00000100:00000010:2.0:1788204666.012203:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00000100:00000001:2.0:1788204666.524206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204666.524211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204666.524215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204666.524217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204666.524219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204666.524224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204666.524225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204666.524229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204666.524229:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204666.524233:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204666.524236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204666.524236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204666.524239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204666.524240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204666.524244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204666.524245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204666.524247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204667.548176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204667.548183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204667.548183:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204667.548186:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204667.548189:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:2.0:1788204667.548192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204667.548193:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204667.548198:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204667.548199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204667.548203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204667.548204:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204667.548208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204667.548213:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204667.548217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204667.548218:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204667.548221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204667.548225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204667.548226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204667.548229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204667.548234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204667.548235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204667.548239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204667.548241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204668.572164:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204668.572172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204668.572175:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204668.572180:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204668.572185:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204668.572189:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204668.572194:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204668.572199:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204668.572203:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204668.572289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204668.572292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204668.572295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204668.572296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204668.572299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204668.572300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204668.572303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204668.572304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204669.596185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204669.596190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204669.596193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204669.596196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204669.596197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204669.596204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204669.596205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204669.596209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204669.596211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204669.596213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204669.596216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204669.596217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204669.596224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204669.596227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204669.596244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204669.596247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204669.596251:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204669.788845:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204669.788852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204669.789586:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204669.789589:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204669.789862:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204669.789864:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204669.790148:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204669.790150:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204669.790548:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204669.790551:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1788204670.620183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204670.620192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204670.620195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204670.620200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204670.620205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204670.620208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204670.620213:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204670.620217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204670.620224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204670.620305:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204670.620307:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204670.620310:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204670.620311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204670.620313:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204670.620313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204670.620315:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204670.620316:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204671.132171:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321029893, at 4321029952 00000100:00000010:2.0:1788204671.132184:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc780. 00010000:00000001:0.0:1788204671.644214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204671.644221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204671.644222:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204671.644228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204671.644234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204671.644237:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204671.644238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204671.644243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204671.644244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204671.644250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204671.644255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204671.644255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204671.644269:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204671.644275:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204671.644278:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204671.644281:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204671.644284:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204672.668243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204672.668251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204672.668254:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204672.668258:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204672.668266:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204672.668269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204672.668277:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204672.668278:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204672.668284:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204672.668284:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204672.668292:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204672.668293:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204672.668299:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204672.668302:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204672.668305:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668307:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668313:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668316:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204672.668323:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204672.668325:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204672.668326:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668328:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204672.668330:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668332:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668334:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668337:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204672.668339:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204672.668340:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204672.668341:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668343:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204672.668345:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668346:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204672.668349:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204673.692217:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204673.692233:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204673.692241:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204673.692253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204673.692257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204673.692269:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204673.692270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204673.692279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204673.692282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204673.692291:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204673.692292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204673.692296:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204673.692302:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204673.692303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204673.692307:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204673.692318:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204673.692320:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204674.716210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204674.716215:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204674.716217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204674.716227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204674.716227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204674.716235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204674.716236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204674.716240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204674.716244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204674.716248:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204674.716249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204674.716252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204674.716259:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204674.716265:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204674.716269:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204674.716272:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204674.716275:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204674.908849:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204674.908856:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204674.909601:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204674.909604:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204674.909882:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204674.909885:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204674.910178:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204674.910181:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204674.910540:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204674.910542:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788204675.740201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204675.740209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204675.740211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204675.740217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204675.740219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204675.740221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204675.740228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204675.740232:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204675.740239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204675.740244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204675.740245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204675.740253:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204675.740257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204675.740264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204675.740267:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204675.740272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204675.740276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204676.252229:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321035013, at 4321035072 00000100:00000010:2.0:1788204676.252252:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc8a0. 00000100:00000001:2.0:1788204676.764222:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204676.764224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204676.764229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204676.764230:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204676.764233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204676.764237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204676.764238:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204676.764240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204676.764244:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204676.764245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204676.764248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204676.764248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204676.764254:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204676.764255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204676.764258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204676.764259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204676.764263:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204677.788177:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204677.788183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204677.788186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204677.788189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204677.788191:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204677.788195:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204677.788197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204677.788199:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204677.788204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204677.788205:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204677.788208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204677.788209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204677.788210:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204677.788215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204677.788217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204677.788219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204677.788223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204677.788224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204677.788228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204677.788232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204677.788233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204677.788236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204677.788237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204678.812211:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204678.812219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204678.812220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204678.812222:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204678.812225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204678.812229:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204678.812233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204678.812234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204678.812237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204678.812239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204678.812243:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204678.812244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204678.812246:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204678.812248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204678.812252:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204678.812263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204678.812267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204679.836179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204679.836186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204679.836190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204679.836192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204679.836195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204679.836201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204679.836204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204679.836211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204679.836212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204679.836218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204679.836223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204679.836225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204679.836227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204679.836230:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204679.836234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204679.836237:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204679.836238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204680.028787:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204680.028793:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204680.029488:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204680.029490:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204680.029713:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204680.029714:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204680.029927:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204680.029928:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204680.030192:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204680.030195:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788204680.860226:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204680.860231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204680.860236:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204680.860238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204680.860238:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204680.860246:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204680.860247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204680.860249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204680.860252:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204680.860255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204680.860256:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204680.860258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204680.860260:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204680.860263:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204680.860264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204680.860266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204680.860267:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204681.372242:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321040133, at 4321040192 00000100:00000010:2.0:1788204681.372261:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647540. 00000100:00000001:2.0:1788204681.884237:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204681.884243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204681.884247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204681.884251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204681.884251:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204681.884259:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204681.884266:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204681.884267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204681.884269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204681.884272:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204681.884274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204681.884289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204681.884289:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204681.884294:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204681.884298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204681.884298:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204681.884301:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204682.908189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204682.908200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204682.908202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204682.908209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204682.908217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204682.908219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204682.908224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204682.908229:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204682.908231:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204682.908239:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204682.908239:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204682.908244:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204682.908245:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908257:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204682.908257:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908270:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908276:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908279:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204682.908285:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204682.908287:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204682.908289:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908293:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204682.908295:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908297:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908300:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204682.908305:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204682.908305:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204682.908306:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908307:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204682.908308:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908309:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204682.908310:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204683.932205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204683.932209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204683.932213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204683.932221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204683.932223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204683.932227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204683.932228:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204683.932234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204683.932235:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204683.932255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204683.932259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204683.932260:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204683.932265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204683.932267:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204683.932271:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204683.932275:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204683.932284:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204684.956239:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204684.956240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204684.956249:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204684.956250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204684.956254:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204684.956260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204684.956260:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204684.956271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204684.956272:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204684.956283:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204684.956286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204684.956288:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204684.956291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204684.956292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204684.956302:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204684.956306:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204684.956311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204685.148215:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204685.148245:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204685.148248:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204685.148252:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204685.148256:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204685.148263:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26371 00000400:00000200:2.0:1788204685.148274:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204685.148281:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204685.148284:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204685.148287:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148290:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.148292:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148293:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204685.148296:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204685.148298:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204685.148300:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204685.148303:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204685.148310:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204685.148312:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204685.148314:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148316:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.148317:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148318:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204685.148320:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204685.148322:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148323:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204685.148326:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204685.148328:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204685.148330:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148331:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204685.148333:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204685.148341:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204685.148343:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204685.148346:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204685.148347:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148349:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204685.148351:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204685.148354:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204685.148360:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204685.148367:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204685.148377:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543458944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204685.148388:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204685.148393:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204685.148398:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543458944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204685.148420:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204685.148423:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204685.148427:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204685.148429:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204685.148430:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204685.148433:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204685.148436:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26371 00000400:00000200:2.0:1788204685.148441:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204685.148444:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204685.148446:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204685.148448:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148450:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.148453:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148454:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204685.148456:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204685.148456:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204685.148457:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204685.148460:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204685.148462:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204685.148464:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204685.148467:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148470:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.148473:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148475:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204685.148477:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204685.148481:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148484:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204685.148491:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204685.148499:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204685.148502:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148505:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204685.148512:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204685.148516:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148519:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204685.148525:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204685.148529:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204685.148531:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204685.148532:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148535:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204685.148537:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204685.148539:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204685.148543:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204685.148547:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204685.148551:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543459072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204685.148561:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204685.148564:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204685.148567:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543459072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204685.148573:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204685.148577:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204685.148580:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204685.148582:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204685.148584:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204685.148587:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204685.148588:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26371 00000400:00000200:2.0:1788204685.148592:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204685.148596:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204685.148599:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204685.148605:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148609:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.148613:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148618:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204685.148622:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204685.148625:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204685.148629:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204685.148633:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204685.148636:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204685.148640:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204685.148646:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148649:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.148653:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148655:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204685.148656:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204685.148658:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148665:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204685.148668:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204685.148672:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204685.148677:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148682:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204685.148686:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204685.148692:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148696:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204685.148712:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204685.148715:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204685.148718:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204685.148720:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148722:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204685.148723:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204685.148725:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204685.148729:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204685.148733:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204685.148739:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543459200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1788204685.148753:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204685.148758:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788204685.148767:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204685.148772:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204685.148780:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543459200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204685.148795:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204685.148798:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204685.148804:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204685.148806:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204685.148809:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204685.148813:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204685.148817:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26371 00000400:00000200:2.0:1788204685.148823:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204685.148827:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204685.148832:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204685.148838:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148842:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.148844:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148845:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204685.148846:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204685.148848:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204685.148851:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204685.148854:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204685.148856:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204685.148858:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204685.148860:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148862:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.148864:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148867:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204685.148870:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204685.148873:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204685.148876:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204685.148877:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204685.148880:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204685.148882:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148883:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204685.148885:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204685.148890:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148891:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204685.148894:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204685.148895:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204685.148899:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204685.148900:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.148902:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204685.148903:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204685.148906:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204685.148911:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204685.148915:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204685.148922:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543459328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204685.148931:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204685.148935:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204685.148943:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543459328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204685.148955:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204685.148962:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204685.148969:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204685.148973:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204685.148976:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204685.148983:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204685.148989:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26371 00000400:00000200:2.0:1788204685.148995:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204685.149000:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204685.149004:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204685.149010:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.149014:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.149018:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.149022:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204685.149026:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204685.149028:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204685.149031:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204685.149033:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204685.149035:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204685.149039:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204685.149043:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.149047:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204685.149049:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204685.149052:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204685.149053:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204685.149057:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204685.149060:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204685.149062:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204685.149067:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204685.149072:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.149077:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204685.149081:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204685.149087:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.149128:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204685.149133:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204685.149137:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204685.149142:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204685.149145:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.149149:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204685.149152:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204685.149155:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204685.149161:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204685.149168:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204685.149178:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543459456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204685.149194:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204685.149199:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204685.149209:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543459456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204685.149241:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204685.149248:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26381) 00000800:00000200:1.1:1788204685.149313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204685.149316:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204685.149479:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204685.149481:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204685.149722:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204685.149724:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204685.150040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204685.150043:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788204685.980200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204685.980203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204685.980217:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204685.980221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.980221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204685.980229:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204685.980230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204685.980233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.980235:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204685.980240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204685.980240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204685.980244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.980245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204685.980250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204685.980251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204685.980254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204685.980255:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204686.492171:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321045253, at 4321045312 00000100:00000010:2.0:1788204686.492185:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00010000:00000001:0.0:1788204687.004201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204687.004201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204687.004209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204687.004210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204687.004213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204687.004217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204687.004219:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204687.004221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204687.004225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204687.004228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204687.004230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204687.004231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204687.004235:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204687.004238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204687.004239:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204687.004242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204687.004245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204688.028206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204688.028212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204688.028216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204688.028219:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204688.028220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204688.028225:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204688.028227:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204688.028230:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204688.028234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204688.028235:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204688.028241:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204688.028241:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204688.028247:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204688.028248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204688.028251:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204688.028253:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204688.028258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204688.028260:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204688.028261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204688.028269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204688.028271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204688.028274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204688.028276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204689.052186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204689.052190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204689.052194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204689.052196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204689.052200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204689.052201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204689.052204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204689.052206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204689.052231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204689.052235:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204689.052237:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204689.052239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204689.052241:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204689.052243:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204689.052244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204689.052246:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204689.052247:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204690.076218:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204690.076222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204690.076226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204690.076227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204690.076231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204690.076235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204690.076237:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204690.076238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204690.076245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204690.076249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204690.076252:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204690.076255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204690.076258:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204690.076259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204690.076265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204690.076270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204690.076278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204690.268869:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204690.268874:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204690.269622:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204690.269624:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204690.270056:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204690.270061:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204690.270555:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204690.270556:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204690.270748:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204690.270750:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788204691.100317:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204691.100320:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204691.100330:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204691.100331:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204691.100338:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204691.100343:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204691.100345:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204691.100347:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204691.100352:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204691.100359:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204691.100360:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204691.100365:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204691.100366:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204691.100375:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204691.100378:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204691.100385:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204691.100386:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204691.612221:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321050373, at 4321050432 00000100:00000010:2.0:1788204691.612243:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc600. 00000100:00000001:2.0:1788204692.124237:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204692.124247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204692.124248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204692.124250:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204692.124253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204692.124266:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204692.124267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204692.124272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204692.124280:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204692.124281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204692.124284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204692.124286:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204692.124291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204692.124293:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204692.124294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204692.124299:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204692.124306:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204693.148233:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204693.148243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204693.148244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204693.148246:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204693.148256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204693.148264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204693.148272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204693.148272:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204693.148280:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204693.148280:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204693.148284:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204693.148285:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204693.148290:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148292:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148298:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204693.148301:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148303:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148306:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204693.148312:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204693.148314:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204693.148317:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148319:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204693.148320:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148322:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148324:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148328:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204693.148330:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204693.148332:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204693.148333:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148335:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204693.148336:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148337:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204693.148339:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204694.172215:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204694.172220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204694.172226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204694.172228:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204694.172234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204694.172236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204694.172239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204694.172243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204694.172249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204694.172252:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204694.172254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204694.172258:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204694.172265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204694.172267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204694.172276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204694.172279:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204694.172289:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.196242:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204695.196245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204695.196251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.196251:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204695.196256:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204695.196260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204695.196264:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204695.196265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204695.196274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204695.196274:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204695.196285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.196286:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204695.196298:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204695.196299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204695.196304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.196304:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204695.196311:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204695.388250:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204695.388260:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204695.388263:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204695.388272:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204695.388277:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204695.388284:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26381 00000400:00000200:2.0:1788204695.388295:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204695.388302:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204695.388305:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204695.388309:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388312:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.388315:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388317:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204695.388320:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204695.388321:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204695.388324:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204695.388327:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204695.388328:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204695.388330:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204695.388331:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388333:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.388334:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388335:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204695.388336:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204695.388338:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388340:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204695.388342:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204695.388345:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204695.388347:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.388348:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204695.388350:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204695.388357:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204695.388359:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204695.388362:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204695.388364:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.388366:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204695.388368:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204695.388371:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204695.388378:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204695.388383:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204695.388397:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543460224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204695.388404:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204695.388408:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204695.388411:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543460224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204695.388428:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204695.388430:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204695.388433:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204695.388434:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204695.388435:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204695.388437:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204695.388439:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26381 00000400:00000200:2.0:1788204695.388442:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204695.388444:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204695.388446:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204695.388449:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388450:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.388452:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388453:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204695.388456:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204695.388458:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204695.388459:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204695.388461:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204695.388464:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204695.388466:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204695.388469:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388471:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.388475:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388483:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204695.388488:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204695.388498:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388505:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204695.388516:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204695.388528:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204695.388536:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.388542:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204695.388546:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204695.388552:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.388555:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204695.388561:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204695.388573:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204695.388579:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204695.388581:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.388587:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204695.388590:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204695.388594:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204695.388603:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204695.388615:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204695.388626:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543460352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204695.388641:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204695.388644:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204695.388652:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543460352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204695.388674:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204695.388679:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204695.388685:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204695.388689:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204695.388693:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204695.388697:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204695.388703:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26381 00000400:00000200:2.0:1788204695.388720:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204695.388724:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204695.388728:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204695.388735:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388743:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.388745:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388747:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204695.388755:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204695.388757:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204695.388758:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204695.388762:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204695.388765:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204695.388767:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204695.388769:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388774:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.388779:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388782:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204695.388783:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204695.388785:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388787:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204695.388788:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204695.388790:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204695.388793:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.388798:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204695.388800:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204695.388803:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.388804:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204695.388809:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204695.388816:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204695.388818:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204695.388821:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.388828:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204695.388831:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204695.388836:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204695.388843:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204695.388854:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204695.388864:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543460480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204695.388881:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204695.388882:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204695.388888:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543460480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204695.388894:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204695.388895:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204695.388910:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204695.388918:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204695.388923:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204695.388927:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204695.388936:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26381 00000400:00000200:2.0:1788204695.388944:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204695.388947:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204695.388949:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204695.388954:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388957:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.388959:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.388960:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204695.388967:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204695.388969:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204695.388979:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204695.388982:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204695.388990:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204695.388992:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:3.1:1788204695.388998:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788204695.389002:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.389005:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1788204695.389005:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000040:2.0:1788204695.389010:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.389016:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204695.389018:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204695.389028:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204695.389031:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204695.389032:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204695.389033:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204695.389035:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.389036:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204695.389038:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204695.389043:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.389055:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204695.389059:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204695.389062:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204695.389067:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204695.389073:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.389083:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204695.389085:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204695.389087:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204695.389133:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204695.389137:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204695.389143:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543460608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204695.389185:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204695.389192:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204695.389199:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543460608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204695.389210:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204695.389219:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204695.389232:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204695.389239:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204695.389244:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204695.389249:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204695.389262:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26381 00000400:00000200:2.0:1788204695.389275:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204695.389288:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204695.389293:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204695.389299:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.389306:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.389308:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.389310:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204695.389312:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204695.389313:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204695.389314:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204695.389316:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204695.389318:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204695.389320:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204695.389328:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.389330:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204695.389335:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204695.389338:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204695.389344:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204695.389347:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204695.389356:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204695.389368:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204695.389372:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204695.389373:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.389375:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204695.389379:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204695.389385:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.389392:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204695.389402:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204695.389409:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204695.389420:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204695.389425:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204695.389428:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204695.389434:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204695.389439:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204695.389449:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204695.389463:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204695.389476:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543460736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204695.389512:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204695.389521:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204695.389531:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543460736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204695.389576:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204695.389587:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26391) 00000800:00000200:3.1:1788204695.389787:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204695.389789:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204695.390017:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204695.390020:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204695.390235:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204695.390238:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204695.390464:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204695.390466:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788204696.220224:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204696.220229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204696.220235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204696.220236:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204696.220241:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204696.220246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204696.220249:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204696.220251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204696.220258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204696.220263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204696.220267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204696.220271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204696.220275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204696.220277:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204696.220286:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204696.220293:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204696.220298:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204696.732203:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321055494, at 4321055552 00000100:00000010:2.0:1788204696.732214:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc480. 00000100:00000001:2.0:1788204697.244182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204697.244188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204697.244190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204697.244195:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204697.244199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204697.244203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204697.244206:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204697.244208:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204697.244213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204697.244226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204697.244233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204697.244240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204697.244241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204697.244244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204697.244245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204697.244248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204697.244249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.268195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204698.268202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204698.268204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204698.268209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204698.268210:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204698.268214:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204698.268216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204698.268218:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.268223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204698.268224:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204698.268228:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.268228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204698.268234:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.268234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204698.268239:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204698.268239:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204698.268245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204698.268248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204698.268252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204698.268266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204698.268268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204698.268273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204698.268275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204699.292190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204699.292199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204699.292202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204699.292211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204699.292213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204699.292222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204699.292224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204699.292233:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204699.292236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204699.292241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204699.292242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204699.292247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204699.292251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204699.292265:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204699.292268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204699.292274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204699.292277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204700.316185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204700.316186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204700.316192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204700.316194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204700.316198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204700.316201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204700.316205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204700.316209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204700.316212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204700.316213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204700.316219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204700.316221:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204700.316223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204700.316226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204700.316232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204700.316235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204700.316239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204700.508742:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204700.508748:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204700.509507:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204700.509510:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204700.509787:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204700.509790:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204700.510065:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204700.510067:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204700.510438:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204700.510440:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000100:00000001:2.0:1788204701.340184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204701.340190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204701.340190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204701.340192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204701.340195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204701.340197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204701.340202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204701.340203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204701.340207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204701.340207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204701.340213:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204701.340214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204701.340218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204701.340218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204701.340224:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204701.340225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204701.340228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204701.852190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321060613, at 4321060672 00000100:00000010:2.0:1788204701.852212:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00000100:00000001:2.0:1788204702.364191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204702.364199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204702.364201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204702.364203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204702.364206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204702.364212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204702.364214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204702.364219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204702.364219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204702.364224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204702.364226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204702.364230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204702.364234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204702.364239:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204702.364242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204702.364244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204702.364245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204703.388213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204703.388219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.388223:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204703.388227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204703.388227:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388232:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204703.388237:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204703.388239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204703.388241:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.388246:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204703.388248:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204703.388252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204703.388261:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.388263:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204703.388272:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.388272:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204703.388282:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204703.388282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204703.388299:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204703.388301:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204703.388304:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388306:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204703.388308:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388310:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388311:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388313:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204703.388315:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204703.388317:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204703.388318:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388319:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204703.388321:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388323:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204703.388325:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204704.412171:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204704.412180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204704.412180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204704.412183:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204704.412186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204704.412189:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204704.412193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204704.412195:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204704.412197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204704.412199:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204704.412202:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204704.412203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204704.412206:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204704.412207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204704.412209:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204704.412213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204704.412224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204705.436190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204705.436195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204705.436200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204705.436202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204705.436206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204705.436206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204705.436210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204705.436212:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204705.436216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204705.436217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204705.436220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204705.436234:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204705.436238:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204705.436241:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204705.436243:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204705.436245:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204705.436248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204705.628834:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204705.629222:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204705.629363:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204705.629500:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204705.630139:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204706.460141:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204706.460147:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204706.460149:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204706.460150:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204706.460153:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204706.460155:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204706.460158:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204706.460160:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204706.460160:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204706.460164:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204706.460166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204706.460169:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204706.460170:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204706.460175:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204706.460182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204706.460182:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204706.460185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204706.972180:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321065733, at 4321065792 00000100:00000010:2.0:1788204706.972194:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00010000:00000001:0.0:1788204707.484205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204707.484205:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204707.484213:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204707.484214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204707.484218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204707.484218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204707.484223:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204707.484226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204707.484228:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204707.484232:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204707.484233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204707.484235:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204707.484239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204707.484240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204707.484245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204707.484249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204707.484250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204707.484252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204707.484258:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204707.484259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204707.484262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204707.484268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204707.484270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.508180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204708.508185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204708.508188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204708.508190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.508191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204708.508202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204708.508209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204708.508210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204708.508213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.508214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204708.508222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204708.508224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204708.508225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204708.508230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.508230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204708.508241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204708.508246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788204708.700290:0:256122:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788204708.700329:0:256122:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788204708.700338:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204708.700343:0:256122:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788204708.700348:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204708.700352:0:256122:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788204708.700356:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204708.700360:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:2.0:1788204708.700365:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204708.700369:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204708.700376:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6d48 00000400:00000010:2.0:1788204708.700379:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6d48. 00000100:00000001:2.0:1788204708.700384:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204708.700394:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39180 x1875055543461504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700418:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204708.700421:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543461504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700426:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.700428:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204708.700436:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204708.700437:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97a00. 00000400:00000200:2.0:1788204708.700439:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204708.700442:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204708.700444:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:2.0:1788204708.700445:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:2.0:1788204708.700447:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204708.700450:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a680 x1875055543461632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700456:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204708.700458:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543461632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700462:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.700463:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204708.700466:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204708.700468:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:2.0:1788204708.700470:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204708.700472:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204708.700474:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6330 00000400:00000010:2.0:1788204708.700475:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6330. 00000100:00000001:2.0:1788204708.700478:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204708.700483:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38a80 x1875055543461760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700492:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204708.700494:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543461760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700508:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.700510:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204708.700519:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204708.700523:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000400:00000200:2.0:1788204708.700530:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204708.700538:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204708.700546:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6990 00000400:00000010:2.0:1788204708.700548:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6990. 00000100:00000001:2.0:1788204708.700552:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204708.700566:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3ad80 x1875055543461888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700638:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204708.700644:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543461888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700658:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.700664:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788204708.700678:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788204708.700684:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000400:00000200:2.0:1788204708.700690:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788204708.700702:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788204708.700712:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6660 00000400:00000010:2.0:1788204708.700719:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6660. 00000100:00000001:2.0:1788204708.700726:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788204708.700739:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a300 x1875055543462016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700763:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204708.700774:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543462016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204730 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204708.700812:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204708.700829:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204709.532174:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204709.532181:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204709.532183:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204709.532188:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204709.532192:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204709.532195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204709.532198:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204709.532201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204709.532213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204709.532302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204709.532304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204709.532307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204709.532308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204709.532310:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204709.532311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204709.532313:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204709.532314:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204710.556216:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204710.556223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204710.556223:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204710.556232:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204710.556235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204710.556241:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204710.556244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204710.556249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204710.556249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204710.556258:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204710.556260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204710.556265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204710.556266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204710.556273:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204710.556278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204710.556279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204710.556287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204710.748695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204710.748702:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204710.749427:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204710.749429:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204710.749686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204710.749689:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204710.749959:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204710.749962:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204710.750325:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204710.750327:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788204711.580145:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204711.580153:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204711.580156:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204711.580161:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204711.580165:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204711.580169:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204711.580172:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204711.580175:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204711.580178:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204711.580179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204711.580184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204711.580189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204711.580191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204711.580195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204711.580196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204711.580208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204711.580209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204712.092230:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321070853, at 4321070912 00000100:00000010:2.0:1788204712.092256:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647540. 00000100:00000001:2.0:1788204712.604197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204712.604207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204712.604209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204712.604212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204712.604219:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204712.604228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204712.604229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204712.604235:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204712.604235:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204712.604241:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204712.604245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204712.604247:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204712.604255:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204712.604256:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204712.604261:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204712.604262:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604271:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604275:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204712.604281:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204712.604285:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204712.604286:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604289:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204712.604293:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604295:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604297:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204712.604301:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204712.604302:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204712.604303:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604304:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204712.604304:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604306:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204712.604307:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204713.628237:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204713.628240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204713.628247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204713.628248:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204713.628252:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204713.628260:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204713.628264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204713.628273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204713.628273:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204713.628282:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204713.628286:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204713.628287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204713.628290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204713.628291:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204713.628295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204713.628296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204713.628299:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.652214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204714.652221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204714.652223:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204714.652228:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204714.652229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204714.652235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204714.652240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204714.652241:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204714.652244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.652247:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204714.652249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204714.652251:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204714.652253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.652256:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204714.652259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204714.652262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204714.652262:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204715.676208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204715.676219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204715.676219:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204715.676222:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204715.676224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204715.676232:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204715.676237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204715.676238:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204715.676243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204715.676243:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204715.676251:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204715.676256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204715.676256:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204715.676262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204715.676265:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204715.676271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204715.676277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204715.868965:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204715.868971:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204715.870001:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204715.870007:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204715.870313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204715.870315:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204715.870534:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204715.870537:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204715.870748:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204715.870750:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:2.0:1788204716.700182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204716.700190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204716.700192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204716.700196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204716.700196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204716.700203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204716.700204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204716.700208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204716.700209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204716.700214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204716.700215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204716.700218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204716.700218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204716.700227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204716.700228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204716.700232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204716.700234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204717.212163:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321075974, at 4321076032 00000100:00000010:2.0:1788204717.212179:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcd20. 00000100:00000001:2.0:1788204717.724221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204717.724228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204717.724234:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204717.724234:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204717.724238:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204717.724240:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204717.724242:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204717.724248:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:2.0:1788204717.724249:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204717.724251:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204717.724256:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204717.724259:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204717.724262:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204717.724266:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204717.724269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204717.724272:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204717.724273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204717.724282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204717.724285:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204717.724286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204717.724297:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204717.724299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204717.724301:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204718.748171:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204718.748179:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204718.748182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204718.748187:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204718.748191:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204718.748195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204718.748198:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204718.748202:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204718.748205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204718.748221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204718.748230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204718.748246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204718.748251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204718.748257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204718.748259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204718.748266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204718.748268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.772224:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204719.772237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204719.772237:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204719.772245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.772249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204719.772251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204719.772260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204719.772264:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204719.772268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204719.772272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.772272:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204719.772279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204719.772280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204719.772288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204719.772293:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204719.772304:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204719.772309:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204720.796195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204720.796199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204720.796205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204720.796209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204720.796209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204720.796221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204720.796221:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204720.796228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204720.796228:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204720.796246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204720.796246:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204720.796250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204720.796253:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204720.796259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204720.796263:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204720.796266:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204720.796270:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204720.988830:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204720.988837:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204720.989466:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204720.989469:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204720.989769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204720.989771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204720.990035:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204720.990038:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204720.990304:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204720.990307:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000001:2.0:1788204721.820159:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204721.820166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204721.820166:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204721.820170:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204721.820171:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204721.820177:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204721.820178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204721.820180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204721.820183:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204721.820185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204721.820188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204721.820188:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204721.820191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204721.820193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204721.820198:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204721.820200:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204721.820204:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204722.332190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321081093, at 4321081152 00000100:00000010:2.0:1788204722.332207:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00000100:00000001:2.0:1788204722.844178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204722.844185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204722.844187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204722.844189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204722.844197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204722.844199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204722.844204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204722.844205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204722.844209:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204722.844209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204722.844214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204722.844218:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204722.844218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204722.844225:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204722.844226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844229:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844238:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204722.844243:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204722.844245:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204722.844248:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844250:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204722.844252:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844254:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844256:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204722.844261:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204722.844262:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204722.844263:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844265:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204722.844266:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844267:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204722.844269:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204723.868175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204723.868183:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204723.868185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204723.868190:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788204723.868194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204723.868197:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204723.868200:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204723.868203:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204723.868206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204723.868293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204723.868297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204723.868302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204723.868303:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204723.868306:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204723.868307:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204723.868310:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204723.868312:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.892190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204724.892193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204724.892197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.892197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204724.892201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204724.892203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204724.892205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204724.892207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204724.892209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204724.892213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204724.892214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204724.892216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204724.892220:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204724.892222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204724.892225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204724.892226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204724.892228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204725.916436:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204725.916439:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204725.916442:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204725.916443:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204725.916444:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204725.916445:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204725.916448:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204725.916448:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204726.108211:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204726.108221:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204726.108224:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204726.108227:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204726.108231:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204726.108239:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26412 00000400:00000200:0.0:1788204726.108249:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204726.108256:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204726.108260:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204726.108263:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108266:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.108268:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108269:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204726.108274:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204726.108276:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204726.108278:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204726.108281:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204726.108283:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204726.108286:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204726.108288:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108290:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.108291:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108293:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204726.108295:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204726.108298:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108300:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204726.108303:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204726.108306:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204726.108309:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108311:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204726.108314:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204726.108323:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204726.108326:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204726.108330:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204726.108331:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108334:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204726.108337:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204726.108350:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204726.108359:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788204726.108365:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204726.108373:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543464064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204726.108381:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204726.108384:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204726.108387:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543464064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204726.108403:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204726.108405:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204726.108408:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204726.108409:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204726.108410:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204726.108411:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204726.108413:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26412 00000400:00000200:0.0:1788204726.108416:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204726.108418:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204726.108420:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204726.108424:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108426:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.108429:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108431:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204726.108436:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204726.108439:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204726.108443:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204726.108447:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204726.108452:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204726.108458:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204726.108466:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108471:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.108475:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108478:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204726.108484:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204726.108488:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108492:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204726.108496:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204726.108501:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204726.108512:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108527:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204726.108531:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204726.108534:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108535:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204726.108540:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204726.108544:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204726.108547:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204726.108549:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108552:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204726.108553:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204726.108555:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204726.108559:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788204726.108563:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204726.108569:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543464192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204726.108578:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204726.108581:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204726.108587:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543464192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204726.108596:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204726.108606:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204726.108611:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204726.108613:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204726.108614:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204726.108618:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204726.108622:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26412 00000400:00000200:0.0:1788204726.108626:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204726.108628:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204726.108630:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204726.108634:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108638:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.108640:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108643:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204726.108648:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204726.108652:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204726.108655:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204726.108660:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204726.108664:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204726.108670:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204726.108678:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108682:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.108685:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108686:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204726.108688:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204726.108694:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108704:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204726.108708:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204726.108713:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204726.108716:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108718:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204726.108721:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204726.108724:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108726:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204726.108731:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204726.108733:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204726.108737:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204726.108738:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108741:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204726.108743:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204726.108745:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204726.108749:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000800:00000200:1.1:1788204726.108753:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204726.108754:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204726.108759:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00100000:0.0:1788204726.108763:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543464320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204726.108790:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204726.108795:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204726.108803:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543464320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204726.108837:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204726.108842:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204726.108848:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204726.108851:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204726.108855:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204726.108858:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204726.108863:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26412 00000400:00000200:0.0:1788204726.108872:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204726.108878:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204726.108881:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204726.108886:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108888:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.108892:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108895:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204726.108897:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204726.108898:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204726.108900:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204726.108901:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204726.108903:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204726.108906:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204726.108912:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108915:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.108919:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108923:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204726.108929:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204726.108936:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204726.108942:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204726.108946:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204726.108949:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204726.108955:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108960:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204726.108963:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204726.108968:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108973:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204726.108979:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204726.108985:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204726.108989:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204726.108993:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.108997:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204726.109000:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204726.109003:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204726.109010:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204726.109016:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204726.109027:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543464448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204726.109039:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204726.109044:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204726.109052:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543464448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204726.109083:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204726.109088:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204726.109116:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204726.109118:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204726.109122:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204726.109126:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204726.109129:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26412 00000400:00000200:0.0:1788204726.109136:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204726.109142:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204726.109145:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204726.109150:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.109152:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.109157:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.109161:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204726.109165:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204726.109168:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204726.109170:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204726.109173:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204726.109175:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204726.109176:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204726.109178:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.109181:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204726.109184:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204726.109187:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204726.109190:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204726.109195:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204726.109200:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204726.109203:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204726.109207:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204726.109211:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.109213:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204726.109216:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204726.109221:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.109225:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204726.109230:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204726.109233:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204726.109238:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204726.109240:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204726.109242:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204726.109244:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204726.109247:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204726.109252:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204726.109256:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204726.109263:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543464576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204726.109274:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204726.109277:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204726.109284:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543464576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204726.109298:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204726.109304:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26422) 00000800:00000200:1.1:1788204726.109344:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204726.109345:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204726.109507:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204726.109508:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204726.109731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204726.109733:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204726.109999:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204726.110001:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1788204726.940200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204726.940205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204726.940213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204726.940217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204726.940222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204726.940223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204726.940229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204726.940231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204726.940282:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204726.940290:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204726.940293:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204727.452195:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321086213, at 4321086272 00000100:00000010:2.0:1788204727.452204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647660. 00010000:00000001:0.0:1788204727.964213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204727.964218:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204727.964222:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204727.964226:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204727.964231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204727.964234:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204727.964237:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204727.964239:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204727.964249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204727.964253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204727.964259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204727.964262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204727.964269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204727.964271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204728.988197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204728.988202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204728.988209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204728.988211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204728.988216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204728.988218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204728.988223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204728.988225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204728.988264:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204728.988267:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204728.988269:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204730.012204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204730.012211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204730.012220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204730.012224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204730.012229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204730.012233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204730.012239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204730.012242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204730.012333:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204730.012336:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204730.012339:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204731.036272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204731.036283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204731.036292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204731.036295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204731.036299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204731.036303:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204731.036315:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204731.036319:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204731.036360:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204731.036364:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204731.036368:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204731.228233:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204731.228247:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204731.228250:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204731.228254:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204731.228260:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204731.228268:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26417 00000400:00000200:2.0:1788204731.228279:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204731.228287:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204731.228291:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204731.228295:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228299:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.228301:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228308:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204731.228313:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204731.228316:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204731.228319:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204731.228322:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204731.228325:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204731.228328:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204731.228331:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228333:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.228335:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228337:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204731.228340:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204731.228344:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228346:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204731.228349:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204731.228354:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204731.228357:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.228360:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204731.228363:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204731.228374:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204731.228378:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204731.228382:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204731.228384:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.228387:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204731.228391:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204731.228395:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204731.228403:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204731.228411:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204731.228420:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543464704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204731.228469:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204731.228474:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204731.228479:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543464704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204731.228503:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204731.228506:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204731.228510:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204731.228511:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204731.228513:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204731.228516:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204731.228518:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26417 00000400:00000200:2.0:1788204731.228523:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204731.228527:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204731.228530:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204731.228533:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228536:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.228539:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228544:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204731.228554:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204731.228560:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204731.228567:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204731.228572:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204731.228578:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204731.228587:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204731.228600:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228612:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.228618:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228626:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204731.228643:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204731.228652:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228656:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204731.228658:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204731.228665:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204731.228670:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.228674:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204731.228680:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204731.228689:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.228692:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204731.228697:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204731.228702:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204731.228705:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204731.228707:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.228712:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204731.228716:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204731.228720:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204731.228728:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204731.228736:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204731.228746:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543464832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204731.228766:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204731.228771:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204731.228780:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543464832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204731.228819:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204731.228827:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204731.228837:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204731.228843:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204731.228847:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204731.228857:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204731.228866:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26417 00000400:00000200:2.0:1788204731.228889:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204731.228898:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204731.228909:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204731.228918:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228923:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.228930:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.228941:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.1:1788204731.228946:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788204731.228951:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:1.1:1788204731.228951:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000020:00000040:2.0:1788204731.228958:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204731.228965:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204731.228971:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204731.228980:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204731.228988:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204731.229002:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229011:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.229020:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229029:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204731.229042:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204731.229055:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229068:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204731.229074:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204731.229080:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204731.229087:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.229136:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204731.229142:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204731.229147:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.229152:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204731.229163:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204731.229170:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204731.229179:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204731.229185:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.229192:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204731.229196:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204731.229202:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204731.229210:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204731.229220:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204731.229237:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543464960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204731.229307:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204731.229311:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204731.229320:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543464960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204731.229353:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204731.229362:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204731.229371:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204731.229378:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204731.229386:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204731.229397:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204731.229408:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26417 00000400:00000200:2.0:1788204731.229439:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204731.229451:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204731.229464:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204731.229474:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229483:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.229494:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229502:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204731.229509:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204731.229516:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204731.229523:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204731.229530:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204731.229539:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204731.229546:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204731.229557:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229562:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.229571:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229580:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204731.229593:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204731.229604:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229616:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204731.229625:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204731.229633:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204731.229643:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.229651:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204731.229656:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204731.229665:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.229670:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204731.229679:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204731.229686:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204731.229698:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204731.229702:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.229712:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204731.229716:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204731.229721:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204731.229729:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204731.229736:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204731.229748:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543465088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204731.229785:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204731.229792:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204731.229804:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543465088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204731.229844:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204731.229844:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204731.229853:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00080000:2.0:1788204731.229854:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204731.229865:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204731.229869:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204731.229872:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204731.229877:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204731.229881:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26417 00000400:00000200:2.0:1788204731.229887:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204731.229890:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204731.229893:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204731.229896:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229898:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.229900:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229903:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204731.229909:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204731.229913:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204731.229919:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204731.229930:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204731.229937:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204731.229949:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204731.229964:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229972:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204731.229985:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204731.229995:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204731.230006:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204731.230016:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204731.230025:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204731.230032:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204731.230039:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204731.230047:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.230051:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204731.230055:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204731.230062:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.230064:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204731.230071:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204731.230078:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204731.230088:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204731.230112:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204731.230117:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204731.230121:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204731.230129:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204731.230139:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204731.230146:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204731.230164:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543465216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204731.230205:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204731.230212:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204731.230224:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543465216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204731.230276:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204731.230290:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26427) 00000800:00000200:1.1:1788204731.230396:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204731.230397:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788204731.230633:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204731.230635:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788204731.230794:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204731.230796:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00010000:00000001:0.0:1788204732.060194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204732.060203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204732.060226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204732.060231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204732.060241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204732.060245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204732.060250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204732.060254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204732.060301:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204732.060305:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204732.060308:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204732.572204:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321091334, at 4321091392 00000100:00000010:2.0:1788204732.572221:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00010000:00000001:0.0:1788204733.084239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204733.084247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204733.084267:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204733.084271:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204733.084280:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084295:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204733.084299:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084304:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084307:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204733.084323:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204733.084327:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204733.084329:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084331:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204733.084332:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084334:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084336:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084338:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204733.084339:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204733.084340:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204733.084341:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084342:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204733.084343:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084344:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204733.084345:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204733.084384:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204733.084388:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204733.084392:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204734.108195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204734.108201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204734.108210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204734.108213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204734.108220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204734.108223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204734.108231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204734.108235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204734.108281:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204734.108285:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204734.108288:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204735.132206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204735.132212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204735.132221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204735.132224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204735.132227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204735.132230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204735.132236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204735.132238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204735.132280:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204735.132285:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204735.132289:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204736.156228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204736.156233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204736.156242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204736.156246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204736.156254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204736.156258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204736.156263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204736.156266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204736.156312:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204736.156316:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204736.156319:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204736.348920:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204736.348928:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204736.349518:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204736.349520:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204736.349735:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204736.349737:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204736.349928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204736.349930:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:3.1:1788204736.350171:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204736.350173:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00010000:00000001:0.0:1788204737.180219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204737.180225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204737.180235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204737.180239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204737.180243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204737.180246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204737.180252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204737.180255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204737.180306:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204737.180311:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204737.180315:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204737.692219:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321096453, at 4321096512 00000100:00000010:2.0:1788204737.692241:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf60. 00010000:00000001:0.0:1788204738.204229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204738.204237:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204738.204242:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204738.204246:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204738.204253:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204738.204256:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204738.204261:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204738.204267:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204738.204282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204738.204285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204738.204295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204738.204297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204738.204300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204738.204302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204738.204369:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204738.204374:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204738.204377:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204739.228222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204739.228229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204739.228242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204739.228247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204739.228257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204739.228262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204739.228271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204739.228275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204739.228329:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204739.228335:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204739.228340:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204740.252266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204740.252285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204740.252295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204740.252298:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204740.252306:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204740.252311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204740.252316:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204740.252320:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204740.252370:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204740.252376:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204740.252379:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204741.276219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204741.276226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204741.276241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204741.276245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204741.276261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204741.276265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204741.276271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204741.276275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204741.276344:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204741.276347:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204741.276349:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204741.276357:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788204741.276360:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204741.276362:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204741.276364:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204741.276366:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204741.276368:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204741.276390:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204741.276395:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204741.276397:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.1:1788204741.468952:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204741.468960:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204741.469666:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204741.469670:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204741.470112:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204741.470114:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204741.471930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204741.471932:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:3.1:1788204741.472144:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204741.472146:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000100:00000001:0.0:1788204742.300248:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204742.300257:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204742.300261:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204742.300270:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788204742.300277:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204742.300282:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204742.300286:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204742.300290:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204742.300294:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204742.300325:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204742.300327:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204742.300331:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204742.300332:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204742.300335:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204742.300336:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204742.300339:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204742.300340:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204742.300396:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204742.300400:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204742.300404:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204742.812205:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321101573, at 4321101632 00000100:00000010:2.0:1788204742.812216:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc300. 00010000:00000001:0.0:1788204743.324288:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204743.324294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204743.324307:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204743.324309:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204743.324311:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204743.324317:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204743.324317:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204743.324323:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324326:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:2.0:1788204743.324329:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204743.324332:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204743.324335:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204743.324340:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204743.324341:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204743.324348:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204743.324348:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204743.324352:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204743.324357:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324358:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204743.324364:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204743.324367:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204743.324369:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324372:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204743.324374:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324376:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324379:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324383:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204743.324385:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204743.324386:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204743.324387:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324388:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204743.324390:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324392:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204743.324394:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204743.324425:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204743.324430:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204743.324434:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204744.348236:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204744.348245:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204744.348249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204744.348249:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204744.348256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204744.348264:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204744.348265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204744.348269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204744.348282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204744.348283:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204744.348287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.348290:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204744.348294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204744.348296:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204744.348298:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204744.348301:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204744.348306:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204745.372192:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204745.372203:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204745.372205:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204745.372210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204745.372211:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204745.372217:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204745.372218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204745.372222:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204745.372226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204745.372226:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204745.372230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204745.372233:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204745.372234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204745.372237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204745.372240:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204745.372247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204745.372250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204745.372298:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204745.372304:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204745.372311:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204746.396222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204746.396229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204746.396237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204746.396240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204746.396250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204746.396254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204746.396255:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204746.396261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204746.396267:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204746.396268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204746.396276:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204746.396284:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204746.396289:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204746.396293:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204746.396297:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204746.396301:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204746.396305:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204746.589029:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204746.589035:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204746.590011:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204746.590013:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204746.590258:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204746.590261:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204746.590557:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204746.590559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204746.590857:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204746.590859:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788204747.420252:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204747.420259:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204747.420263:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204747.420269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204747.420276:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204747.420281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204747.420283:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204747.420289:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204747.420294:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204747.420295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204747.420303:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204747.420304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204747.420315:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204747.420320:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204747.420324:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204747.420334:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204747.420336:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204747.932258:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321106694, at 4321106752 00000100:00000010:2.0:1788204747.932285:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcae0. 00010000:00000001:0.0:1788204748.444224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204748.444236:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204748.444238:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204748.444247:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204748.444248:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204748.444252:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204748.444260:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204748.444261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204748.444268:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204748.444276:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204748.444278:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204748.444280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204748.444289:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204748.444294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204748.444296:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204748.444298:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204748.444303:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204748.444305:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204748.444309:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204748.444309:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204748.444317:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204748.444322:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204748.444325:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204748.444372:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204748.444375:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204748.444379:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204749.468219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204749.468219:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204749.468228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204749.468229:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204749.468231:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204749.468236:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204749.468237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204749.468239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204749.468243:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204749.468245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204749.468247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204749.468247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204749.468252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204749.468252:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204749.468255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204749.468257:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204749.468261:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204750.492184:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204750.492194:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204750.492197:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204750.492204:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204750.492210:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204750.492215:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204750.492219:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204750.492222:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204750.492226:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204750.492249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204750.492256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204750.492268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204750.492273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204750.492283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204750.492287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204750.492292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204750.492295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204751.516159:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204751.516168:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204751.516170:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204751.516178:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204751.516184:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204751.516189:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204751.516195:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204751.516199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204751.516199:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204751.516203:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204751.516206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204751.516215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204751.516218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204751.516223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204751.516225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204751.516229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204751.516231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204751.516260:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204751.516262:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204751.516265:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204751.708845:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204751.708853:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204751.709398:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204751.709400:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204751.709666:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204751.709669:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204751.709862:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204751.709864:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204751.710053:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204751.710054:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1788204752.540195:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204752.540205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204752.540209:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204752.540210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204752.540217:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204752.540229:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204752.540231:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204752.540241:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204752.540243:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204752.540246:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540255:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204752.540259:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204752.540260:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204752.540274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204752.540277:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204752.540287:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204752.540289:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204752.540305:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204752.540315:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204752.540323:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204752.540329:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540333:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204752.540336:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540340:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540343:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540348:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204752.540350:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204752.540351:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204752.540353:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540356:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204752.540359:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540362:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204752.540365:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204753.052210:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321111813, at 4321111872 00000100:00000010:2.0:1788204753.052228:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00000100:00000001:2.0:1788204753.564249:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204753.564259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204753.564261:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204753.564265:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204753.564266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204753.564276:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204753.564278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204753.564283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204753.564283:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204753.564290:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204753.564292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204753.564296:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204753.564297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204753.564303:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204753.564305:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204753.564310:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204753.564311:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204754.588219:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204754.588227:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204754.588228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204754.588229:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204754.588232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204754.588235:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204754.588240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204754.588240:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204754.588243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204754.588245:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204754.588247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204754.588251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204754.588252:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204754.588256:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204754.588260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204754.588261:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204754.588262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204754.588289:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204754.588293:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204754.588296:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204755.612206:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204755.612217:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204755.612220:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204755.612222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204755.612228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204755.612230:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204755.612233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204755.612236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204755.612238:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204755.612240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204755.612243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204755.612246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204755.612246:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204755.612248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204755.612254:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204755.612258:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204755.612261:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204756.636170:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204756.636179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204756.636180:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204756.636185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204756.636185:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204756.636192:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204756.636193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204756.636195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204756.636198:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204756.636200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204756.636202:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204756.636203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204756.636209:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204756.636210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204756.636213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204756.636214:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204756.636219:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204756.828690:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204756.828693:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204756.829276:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204756.829278:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204756.829461:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204756.829462:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204756.829620:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204756.829622:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204756.829775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204756.829777:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1788204757.660215:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204757.660220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204757.660224:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204757.660226:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204757.660228:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204757.660229:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204757.660234:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204757.660235:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204757.660241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204757.660241:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204757.660245:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204757.660247:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204757.660248:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204757.660249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204757.660256:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204757.660260:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204757.660262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204757.660265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204757.660267:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204757.660269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204757.660273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204757.660278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204757.660279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204757.660302:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204757.660304:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204757.660307:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204758.172213:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321116933, at 4321116992 00000100:00000010:2.0:1788204758.172233:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc180. 00000100:00000001:2.0:1788204758.684177:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204758.684183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204758.684184:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204758.684187:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204758.684188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204758.684192:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204758.684196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204758.684196:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204758.684201:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204758.684202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204758.684205:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204758.684208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204758.684208:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204758.684212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204758.684213:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204758.684219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204758.684222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204759.708247:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204759.708257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204759.708263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204759.708267:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204759.708273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204759.708281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204759.708281:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204759.708296:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204759.708297:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204759.708300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204759.708309:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204759.708311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204759.708315:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204759.708317:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204759.708321:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204759.708324:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204759.708327:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204760.732190:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204760.732200:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204760.732202:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204760.732207:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204760.732211:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204760.732214:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204760.732219:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204760.732221:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204760.732226:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204760.732229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204760.732236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204760.732241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204760.732245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204760.732250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204760.732253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204760.732256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204760.732261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204760.732304:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204760.732313:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204760.732317:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204761.756169:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204761.756175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204761.756177:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204761.756180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204761.756181:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204761.756188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204761.756188:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204761.756191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204761.756195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204761.756196:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204761.756197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204761.756200:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204761.756202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204761.756204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204761.756212:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204761.756215:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204761.756220:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204761.950371:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204761.950379:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204761.951245:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204761.951254:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204761.951808:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204761.951810:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204761.952133:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204761.952135:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788204761.952408:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204761.952410:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00010000:00000001:0.0:1788204762.780169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204762.780169:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204762.780177:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204762.780178:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204762.780181:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204762.780186:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204762.780188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204762.780190:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204762.780194:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204762.780194:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204762.780199:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204762.780199:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204762.780205:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204762.780206:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204762.780208:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780213:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204762.780224:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780232:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780238:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204762.780259:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204762.780266:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204762.780269:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780273:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204762.780275:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780278:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204762.780284:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204762.780285:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204762.780286:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780287:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204762.780288:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780290:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204762.780292:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204763.292209:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321122055, at 4321122112 00000100:00000010:2.0:1788204763.292225:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00000100:00000001:2.0:1788204763.804225:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204763.804237:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204763.804240:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204763.804252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204763.804252:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204763.804265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204763.804267:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204763.804280:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204763.804282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204763.804286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204763.804298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204763.804304:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204763.804310:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204763.804314:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204763.804315:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204763.804321:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204763.804325:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204763.804351:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204763.804355:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204763.804358:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204764.828235:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204764.828245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204764.828247:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204764.828255:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204764.828256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204764.828275:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204764.828276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204764.828284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204764.828286:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204764.828293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204764.828295:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204764.828304:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204764.828306:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204764.828311:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204764.828318:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204764.828320:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204764.828323:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204765.852225:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204765.852231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204765.852236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204765.852237:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204765.852242:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204765.852245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204765.852249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204765.852251:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204765.852252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204765.852256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204765.852259:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204765.852261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204765.852264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204765.852264:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204765.852272:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204765.852277:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204765.852282:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204766.876233:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204766.876237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204766.876244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204766.876247:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204766.876251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204766.876261:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204766.876262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204766.876281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204766.876282:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204766.876290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204766.876294:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204766.876298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204766.876303:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204766.876303:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204766.876315:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204766.876319:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204766.876324:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204767.068981:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204767.068995:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204767.070044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204767.070047:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204767.070419:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204767.070422:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204767.070752:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204767.070755:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788204767.071120:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204767.071123:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1788204767.900181:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204767.900190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204767.900191:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204767.900196:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204767.900198:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204767.900204:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204767.900207:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204767.900209:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204767.900215:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204767.900216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204767.900221:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204767.900222:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204767.900229:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204767.900229:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204767.900234:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204767.900235:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204767.900238:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204767.900246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204767.900250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204767.900254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204767.900257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204767.900261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204767.900262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204768.412215:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321127174, at 4321127232 00000100:00000010:2.0:1788204768.412231:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00000100:00000001:2.0:1788204768.924189:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204768.924193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204768.924197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204768.924198:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204768.924203:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204768.924205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204768.924209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204768.924210:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204768.924214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204768.924217:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204768.924218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204768.924222:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204768.924225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204768.924226:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204768.924229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204768.924230:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204768.924239:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204769.948169:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204769.948175:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204769.948178:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204769.948186:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204769.948191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204769.948193:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204769.948196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204769.948197:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204769.948201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204769.948202:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204769.948205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204769.948206:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204769.948210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204769.948211:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204769.948213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204769.948218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204769.948220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204769.948239:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204769.948242:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204769.948244:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788204770.972321:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788204770.972325:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788204770.972327:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204770.972330:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788204770.972332:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788204770.972334:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204770.972337:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788204770.972339:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204770.972342:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204770.972363:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204770.972366:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204770.972371:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204770.972373:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204770.972376:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204770.972377:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204770.972381:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204770.972382:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204771.996221:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204771.996222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204771.996227:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204771.996229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204771.996237:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204771.996240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204771.996242:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204771.996245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204771.996248:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204771.996251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204771.996254:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204771.996255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204771.996262:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204771.996265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204771.996268:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204771.996269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204771.996274:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204772.188212:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204772.188225:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204772.188228:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204772.188233:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204772.188238:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204772.188246:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26458 00000400:00000200:2.0:1788204772.188257:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204772.188264:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204772.188269:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204772.188273:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188276:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.188279:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188282:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204772.188286:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204772.188288:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204772.188291:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204772.188295:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204772.188298:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204772.188301:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204772.188305:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188307:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.188310:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188312:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204772.188315:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204772.188319:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188322:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204772.188325:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204772.188329:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204772.188333:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.188335:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204772.188338:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204772.188348:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204772.188353:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204772.188357:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204772.188359:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.188362:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204772.188365:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204772.188369:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204772.188379:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204772.188387:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204772.188399:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543469824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204772.188411:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204772.188416:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204772.188422:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543469824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204772.188449:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204772.188452:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204772.188456:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204772.188458:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204772.188460:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204772.188463:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204772.188468:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26458 00000400:00000200:2.0:1788204772.188473:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204772.188478:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204772.188481:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204772.188488:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188493:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.188498:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188502:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204772.188509:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204772.188512:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204772.188516:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204772.188522:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204772.188528:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204772.188536:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204772.188545:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188550:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.188556:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188561:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204772.188567:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204772.188574:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188579:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204772.188583:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204772.188591:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204772.188598:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.188605:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204772.188611:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204772.188619:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.188625:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204772.188633:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204772.188639:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204772.188644:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204772.188648:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.188653:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204772.188657:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204772.188661:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204772.188670:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204772.188679:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204772.188691:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543469952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204772.188730:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204772.188736:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204772.188745:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543469952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788204772.188751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204772.188764:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204772.188769:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204772.188777:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204772.188782:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204772.188786:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204772.188791:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204772.188800:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26458 00000400:00000200:2.0:1788204772.188810:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204772.188820:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204772.188826:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204772.188834:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188840:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.188846:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188850:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204772.188853:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204772.188857:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204772.188861:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204772.188865:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204772.188871:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204772.188878:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204772.188887:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188893:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.188899:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188905:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204772.188913:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204772.188921:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204772.188928:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204772.188935:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204772.188943:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204772.188949:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.188956:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204772.188961:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204772.188968:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.188973:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204772.188983:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204772.188989:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204772.188997:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204772.189002:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.189007:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204772.189012:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204772.189017:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204772.189026:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204772.189034:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204772.189057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00100000:2.0:1788204772.189059:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543470080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204772.189073:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204772.189079:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204772.189130:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543470080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204772.189150:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204772.189157:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204772.189166:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204772.189171:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204772.189175:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204772.189182:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204772.189190:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26458 00000400:00000200:2.0:1788204772.189202:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204772.189211:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204772.189217:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204772.189227:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189232:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.189238:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189245:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204772.189248:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204772.189252:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204772.189256:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204772.189260:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204772.189266:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204772.189272:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204772.189280:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189285:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.189291:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189298:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204772.189305:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204772.189313:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189321:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204772.189326:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204772.189333:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204772.189338:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.189345:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204772.189352:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204772.189359:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.189363:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204772.189371:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204772.189378:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204772.189385:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204772.189388:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.189393:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204772.189396:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204772.189400:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204772.189407:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204772.189414:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204772.189424:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543470208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204772.189447:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204772.189453:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204772.189461:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543470208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788204772.189479:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204772.189482:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204772.189489:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204772.189496:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204772.189501:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204772.189505:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204772.189510:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204772.189517:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26458 00000400:00000200:2.0:1788204772.189527:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204772.189534:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204772.189540:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204772.189548:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189554:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.189559:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189566:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204772.189571:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204772.189575:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204772.189579:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204772.189582:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204772.189586:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204772.189592:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204772.189600:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189605:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204772.189610:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189616:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204772.189621:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204772.189626:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204772.189630:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204772.189636:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204772.189641:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204772.189645:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.189649:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204772.189654:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204772.189658:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.189662:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204772.189667:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204772.189673:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204772.189680:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204772.189683:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204772.189686:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204772.189690:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204772.189693:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204772.189698:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204772.189704:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204772.189714:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543470336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204772.189752:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204772.189757:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204772.189767:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543470336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204772.189787:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204772.189795:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26468) 00000800:00000200:1.1:1788204772.189849:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204772.190017:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204773.020156:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204773.020165:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204773.020167:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204773.020174:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204773.020176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204773.020182:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204773.020187:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204773.020188:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204773.020190:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204773.020193:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204773.020194:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020199:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204773.020199:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204773.020202:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020205:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204773.020205:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020212:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204773.020217:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204773.020218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204773.020219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020220:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204773.020221:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020223:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020225:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204773.020228:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204773.020229:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204773.020230:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204773.020232:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204773.020234:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204773.020273:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204773.020276:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204773.020278:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204773.532183:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321132293, at 4321132352 00000100:00000010:2.0:1788204773.532203:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00010000:00000001:0.0:1788204774.044226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204774.044231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204774.044239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204774.044242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204774.044244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204774.044246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204774.044252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204774.044255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204774.044367:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204774.044371:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204774.044373:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204774.044376:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204774.044378:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204774.044381:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204774.044384:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204774.044386:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204774.044388:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.068184:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204775.068188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204775.068194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204775.068201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204775.068203:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204775.068204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.068208:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204775.068212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204775.068213:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204775.068215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.068219:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204775.068222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204775.068225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204775.068226:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204775.068232:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204775.068237:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204775.068242:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788204775.260411:0:256123:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:0.0:1788204775.260460:0:256123:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204775.260470:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204775.260475:0:256123:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:0.0:1788204775.260481:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204775.260485:0:256123:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:0.0:1788204775.260491:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204775.260494:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:0.0:1788204775.260500:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204775.260505:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204775.260512:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6c38 00000400:00000010:0.0:1788204775.260516:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6c38. 00000100:00000001:0.0:1788204775.260522:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204775.260544:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75500 x1875055543469824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260574:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204775.260579:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75500 x1875055543469824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260587:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.260590:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204775.260600:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204775.260603:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:0.0:1788204775.260607:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204775.260611:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204775.260614:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6aa0 00000400:00000010:0.0:1788204775.260616:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6aa0. 00000100:00000001:0.0:1788204775.260619:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204775.260624:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74380 x1875055543469952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260633:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204775.260636:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74380 x1875055543469952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260643:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.260645:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204775.260649:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204775.260652:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000400:00000200:0.0:1788204775.260655:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204775.260658:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204775.260662:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6660 00000400:00000010:0.0:1788204775.260695:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6660. 00000100:00000001:0.0:1788204775.260699:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204775.260704:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d76a00 x1875055543470080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260712:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204775.260716:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543470080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260723:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.260725:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204775.260731:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204775.260733:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000400:00000200:0.0:1788204775.260736:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204775.260739:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204775.260743:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6330 00000400:00000010:0.0:1788204775.260744:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6330. 00000100:00000001:0.0:1788204775.260747:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204775.260752:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74a80 x1875055543470208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260760:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204775.260763:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74a80 x1875055543470208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260770:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.260772:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204775.260777:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204775.260779:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:0.0:1788204775.260782:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204775.260785:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204775.260787:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6d48 00000400:00000010:0.0:1788204775.260789:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6d48. 00000100:00000001:0.0:1788204775.260792:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204775.260796:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d77b80 x1875055543470336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260803:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204775.260807:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543470336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204797 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204775.260813:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204775.260815:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204776.092177:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204776.092185:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204776.092188:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204776.092191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204776.092196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204776.092196:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204776.092202:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204776.092205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204776.092208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204776.092208:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204776.092215:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204776.092216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204776.092220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204776.092220:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204776.092227:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204776.092228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204776.092231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204776.092259:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204776.092262:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204776.092264:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788204777.116207:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204777.116217:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204777.116223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204777.116223:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204777.116234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204777.116236:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204777.116248:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204777.116251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204777.116252:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204777.116256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.116260:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204777.116264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204777.116265:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204777.116270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.116272:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204777.116279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204777.116281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204777.308188:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204777.308198:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204777.308201:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204777.308203:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204777.308208:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204777.308215:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26463 00000400:00000200:2.0:1788204777.308224:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204777.308231:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204777.308235:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204777.308238:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308241:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.308242:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308244:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204777.308248:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204777.308249:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204777.308252:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204777.308254:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204777.308256:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204777.308257:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204777.308259:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308261:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.308262:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308263:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204777.308265:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204777.308267:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308268:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204777.308270:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204777.308273:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204777.308275:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308276:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204777.308278:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204777.308283:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204777.308285:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204777.308287:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204777.308288:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308290:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204777.308292:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204777.308295:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204777.308301:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204777.308306:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204777.308314:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543470464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204777.308326:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204777.308329:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204777.308332:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543470464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204777.308351:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204777.308354:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204777.308357:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204777.308358:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204777.308360:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204777.308363:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204777.308365:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26463 00000400:00000200:2.0:1788204777.308369:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204777.308372:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204777.308374:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204777.308377:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308380:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.308383:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308385:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204777.308388:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204777.308390:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204777.308392:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204777.308394:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204777.308397:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204777.308401:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204777.308406:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308410:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.308414:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308418:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204777.308424:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204777.308431:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308436:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204777.308441:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204777.308448:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204777.308452:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308457:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204777.308463:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204777.308471:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308476:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204777.308480:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204777.308486:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204777.308492:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204777.308494:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308497:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204777.308501:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204777.308505:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204777.308510:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204777.308514:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204777.308523:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543470592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204777.308536:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204777.308540:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204777.308546:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543470592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204777.308563:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204777.308568:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204777.308579:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204777.308583:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204777.308587:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204777.308594:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204777.308601:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26463 00000400:00000200:2.0:1788204777.308609:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204777.308618:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204777.308622:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204777.308640:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308643:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.308646:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308648:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204777.308650:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204777.308651:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204777.308653:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204777.308655:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204777.308657:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204777.308659:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204777.308662:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308663:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.308665:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308667:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204777.308669:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204777.308671:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308674:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204777.308676:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204777.308678:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204777.308681:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308683:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204777.308685:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204777.308688:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308690:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204777.308693:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204777.308696:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204777.308698:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204777.308700:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308702:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204777.308704:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204777.308707:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204777.308711:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204777.308717:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204777.308724:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543470720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204777.308736:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204777.308740:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204777.308746:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543470720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204777.308761:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204777.308766:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204777.308774:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204777.308777:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204777.308780:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204777.308787:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204777.308794:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26463 00000400:00000200:2.0:1788204777.308805:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204777.308811:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204777.308816:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204777.308822:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308828:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.308831:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308836:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.1:1788204777.308839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788204777.308841:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204777.308844:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000800:00000200:1.1:1788204777.308844:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000020:00000001:2.0:1788204777.308848:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204777.308852:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204777.308854:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204777.308860:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204777.308868:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308872:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.308876:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308881:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204777.308886:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204777.308892:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204777.308897:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204777.308903:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204777.308907:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204777.308912:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308918:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204777.308921:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204777.308927:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308930:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204777.308937:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204777.308942:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204777.308948:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204777.308952:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.308957:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204777.308960:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204777.308963:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204777.308969:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204777.308975:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204777.308984:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543470848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204777.308996:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204777.309000:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204777.309008:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543470848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204777.309025:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204777.309031:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204777.309037:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204777.309041:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204777.309044:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204777.309051:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204777.309056:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26463 00000400:00000200:2.0:1788204777.309064:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204777.309069:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204777.309074:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204777.309121:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.309125:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.309128:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.309134:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204777.309140:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204777.309141:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204777.309144:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204777.309149:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204777.309154:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204777.309158:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204777.309164:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.309169:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204777.309174:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204777.309180:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204777.309185:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204777.309192:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204777.309198:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204777.309203:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204777.309209:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204777.309215:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.309220:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204777.309224:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204777.309229:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.309234:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204777.309241:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204777.309246:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204777.309252:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204777.309256:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204777.309260:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204777.309264:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204777.309268:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204777.309275:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204777.309282:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204777.309291:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543470976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204777.309307:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204777.309312:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204777.309321:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543470976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204777.309336:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204777.309343:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26473) 00000800:00000200:1.1:1788204777.309514:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204777.309517:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204777.309829:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204777.309831:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204777.310136:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204777.310139:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204777.310405:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204777.310407:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:2.0:1788204778.140167:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204778.140173:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204778.140176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204778.140176:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204778.140183:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204778.140184:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204778.140188:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204778.140189:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204778.140193:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204778.140194:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204778.140197:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204778.140200:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204778.140202:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204778.140203:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204778.140208:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204778.140210:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204778.140216:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204778.140222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204778.140224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204778.140228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204778.140229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204778.140233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204778.140235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204778.652221:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321137413, at 4321137472 00000100:00000010:2.0:1788204778.652242:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00000100:00000001:2.0:1788204779.164213:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204779.164218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204779.164222:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204779.164223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204779.164227:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204779.164230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204779.164236:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204779.164237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204779.164242:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204779.164245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204779.164247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204779.164247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204779.164254:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204779.164255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204779.164259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204779.164260:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204779.164267:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204780.188234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204780.188242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204780.188250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204780.188252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204780.188256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204780.188258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204780.188263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204780.188265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204780.188427:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204780.188432:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204780.188434:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204780.188438:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204780.188442:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204780.188445:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204780.188448:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204780.188450:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204780.188453:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204781.212251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204781.212254:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204781.212257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204781.212264:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204781.212267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204781.212269:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204781.212272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204781.212285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204781.212285:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204781.212292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204781.212297:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204781.212300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204781.212304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204781.212308:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204781.212319:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204781.212323:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204781.212327:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204782.236227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204782.236227:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204782.236237:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204782.236238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.236240:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204782.236250:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204782.236251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204782.236258:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204782.236262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.236263:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204782.236277:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204782.236279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204782.236284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.236284:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204782.236288:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204782.236298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204782.236301:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204782.428211:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204782.428226:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204782.428229:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204782.428233:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204782.428239:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204782.428247:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26468 00000400:00000200:2.0:1788204782.428259:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204782.428266:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204782.428270:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204782.428274:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428276:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.428279:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428281:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204782.428286:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204782.428288:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204782.428292:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204782.428295:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204782.428298:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204782.428301:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204782.428304:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428306:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.428308:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428309:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204782.428312:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204782.428314:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428316:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204782.428318:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204782.428322:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204782.428324:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.428325:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204782.428328:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204782.428338:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204782.428340:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204782.428344:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204782.428346:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.428349:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204782.428352:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204782.428357:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204782.428369:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204782.428377:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204782.428386:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543471104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204782.428397:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204782.428402:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204782.428407:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543471104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204782.428441:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204782.428444:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204782.428447:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204782.428448:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204782.428449:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204782.428451:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204782.428453:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26468 00000400:00000200:2.0:1788204782.428456:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204782.428458:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204782.428460:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204782.428462:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428463:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.428465:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428466:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204782.428468:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204782.428469:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204782.428470:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204782.428472:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204782.428474:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204782.428475:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204782.428477:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428479:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.428480:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428481:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204782.428483:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204782.428487:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428488:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204782.428490:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204782.428495:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204782.428497:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.428498:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204782.428502:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204782.428504:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.428505:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204782.428508:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204782.428511:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204782.428515:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204782.428516:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.428524:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204782.428528:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204782.428530:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204782.428537:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204782.428555:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204782.428562:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543471232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204782.428621:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204782.428628:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204782.428655:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543471232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204782.428679:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204782.428685:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204782.428697:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204782.428700:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204782.428707:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204782.428714:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204782.428719:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26468 00000400:00000200:2.0:1788204782.428732:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204782.428744:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204782.428748:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204782.428755:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428764:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.428775:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428780:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204782.428790:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204782.428795:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204782.428800:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204782.428803:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204782.428811:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204782.428816:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204782.428827:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428831:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.428833:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428835:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204782.428838:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204782.428845:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204782.428850:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204782.428855:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204782.428870:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204782.428876:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.428881:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204782.428886:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204782.428902:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.428905:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204782.428909:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204782.428914:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204782.428921:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204782.428925:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.428929:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204782.428931:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204782.428934:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204782.428943:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204782.428952:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204782.428962:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204782.428966:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00100000:2.0:1788204782.428969:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543471360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204782.428987:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204782.428992:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204782.428999:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543471360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204782.429012:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204782.429014:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204782.429017:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204782.429018:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204782.429019:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204782.429021:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204782.429023:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26468 00000400:00000200:2.0:1788204782.429027:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204782.429029:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204782.429031:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204782.429033:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429034:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.429035:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429037:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204782.429043:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204782.429046:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204782.429050:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204782.429052:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204782.429054:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204782.429057:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204782.429063:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429066:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.429071:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429073:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204782.429075:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204782.429080:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429085:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204782.429111:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204782.429115:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204782.429117:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.429118:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204782.429120:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204782.429121:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.429123:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204782.429127:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204782.429128:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204782.429130:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204782.429131:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.429132:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204782.429133:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204782.429134:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204782.429137:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204782.429140:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204782.429144:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543471488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204782.429212:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204782.429215:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204782.429224:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543471488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204782.429266:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204782.429273:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204782.429277:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204782.429278:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204782.429280:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204782.429284:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204782.429290:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26468 00000400:00000200:2.0:1788204782.429298:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204782.429307:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204782.429311:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204782.429318:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429324:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.429331:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429336:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204782.429347:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204782.429353:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204782.429358:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204782.429362:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204782.429370:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204782.429380:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204782.429389:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429393:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204782.429398:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429406:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204782.429412:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204782.429421:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204782.429428:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204782.429436:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204782.429444:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204782.429449:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.429454:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204782.429458:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204782.429464:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.429469:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204782.429477:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204782.429482:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204782.429487:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204782.429490:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204782.429499:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204782.429503:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204782.429506:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204782.429513:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204782.429521:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204782.429533:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543471616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204782.429566:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204782.429571:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204782.429586:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543471616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204782.429609:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204782.429617:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26478) 00000800:00000200:1.1:1788204782.429803:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204782.429805:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204782.430040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204782.430041:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204782.430220:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204782.430222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204782.430433:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204782.430434:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788204783.260206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204783.260207:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204783.260211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204783.260214:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204783.260218:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204783.260223:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204783.260226:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204783.260228:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204783.260230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204783.260233:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204783.260235:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204783.260239:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204783.260242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204783.260243:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204783.260246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204783.260248:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260256:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204783.260262:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204783.260264:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204783.260265:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260267:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204783.260268:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260270:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260272:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204783.260278:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204783.260279:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204783.260281:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260283:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204783.260284:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260286:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204783.260288:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204783.772192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321142534, at 4321142592 00000100:00000010:2.0:1788204783.772210:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00000100:00000001:2.0:1788204784.284176:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204784.284177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204784.284180:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204784.284183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204784.284184:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204784.284189:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204784.284192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204784.284193:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204784.284197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204784.284197:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204784.284202:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204784.284205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204784.284205:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204784.284208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204784.284212:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204784.284215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204784.284218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.308189:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204785.308193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204785.308195:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204785.308198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.308199:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204785.308206:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204785.308209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204785.308212:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204785.308214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.308215:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204785.308220:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204785.308221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204785.308223:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204785.308224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204785.308228:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204785.308230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204785.308234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204786.332191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204786.332192:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204786.332197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204786.332198:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204786.332202:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204786.332204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204786.332207:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204786.332209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204786.332211:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204786.332218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204786.332219:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204786.332221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204786.332226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204786.332226:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204786.332230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204786.332231:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204786.332235:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204787.356190:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204787.356193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204787.356199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204787.356202:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204787.356206:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204787.356207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204787.356210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204787.356212:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204787.356215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204787.356218:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204787.356219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204787.356223:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204787.356226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204787.356228:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204787.356229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204787.356232:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204787.356238:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204787.548748:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204787.548754:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204787.549291:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204787.549293:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204787.549496:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204787.549498:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204787.549758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204787.549759:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204787.549979:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204787.549981:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1788204788.380173:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204788.380175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204788.380181:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204788.380181:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204788.380185:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788204788.380185:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204788.380189:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204788.380192:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204788.380193:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204788.380197:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204788.380198:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204788.380202:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204788.380203:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204788.380206:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204788.380209:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204788.380211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204788.380214:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204788.380215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204788.380220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204788.380221:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204788.380222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204788.380227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204788.380228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204788.892177:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321147653, at 4321147712 00000100:00000010:2.0:1788204788.892192:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00000100:00000001:2.0:1788204789.404185:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204789.404187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204789.404192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204789.404192:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204789.404197:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204789.404198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204789.404201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204789.404203:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204789.404204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204789.404206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204789.404209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204789.404209:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204789.404212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204789.404214:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204789.404218:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204789.404220:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204789.404223:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204790.428150:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204790.428157:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204790.428158:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204790.428160:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204790.428161:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204790.428168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788204790.428168:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204790.428171:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204790.428175:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204790.428176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204790.428180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204790.428180:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204790.428185:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204790.428188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204790.428188:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204790.428191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204790.428194:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204791.452175:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204791.452177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204791.452181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204791.452181:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204791.452185:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204791.452188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204791.452190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204791.452194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204791.452197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204791.452197:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204791.452203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204791.452205:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204791.452206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204791.452208:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204791.452214:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204791.452217:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204791.452220:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204792.476184:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204792.476187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204792.476191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204792.476191:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204792.476193:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204792.476197:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204792.476198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204792.476201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204792.476202:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204792.476205:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204792.476208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204792.476209:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204792.476211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204792.476212:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204792.476214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204792.476217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204792.476219:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204792.668733:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204792.668740:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204792.669270:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204792.669272:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204792.669491:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204792.669493:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204792.669699:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204792.669700:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204792.669887:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204792.669889:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000001:2.0:1788204793.500167:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204793.500172:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204793.500175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204793.500176:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204793.500182:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204793.500186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204793.500186:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204793.500192:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788204793.500192:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204793.500195:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204793.500199:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204793.500200:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204793.500204:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204793.500205:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204793.500208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500210:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204793.500212:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500213:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204793.500222:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204793.500223:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204793.500225:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500226:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204793.500228:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500229:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500230:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204793.500235:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204793.500236:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204793.500237:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500239:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204793.500239:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500241:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204793.500242:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204794.012217:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321152773, at 4321152832 00000100:00000010:2.0:1788204794.012234:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647ba0. 00000100:00000001:2.0:1788204794.524223:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204794.524233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204794.524234:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204794.524240:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204794.524242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204794.524248:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204794.524256:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204794.524257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204794.524261:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204794.524264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204794.524266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204794.524275:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204794.524277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204794.524282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204794.524282:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204794.524295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204794.524297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204795.548224:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204795.548232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204795.548236:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204795.548243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204795.548244:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204795.548260:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204795.548261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204795.548268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204795.548270:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204795.548277:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204795.548279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204795.548283:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204795.548287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204795.548290:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204795.548301:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204795.548304:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204795.548310:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.572220:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788204796.572229:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204796.572233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204796.572233:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204796.572245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204796.572246:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788204796.572258:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204796.572262:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204796.572264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204796.572266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204796.572270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204796.572275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204796.572279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204796.572279:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204796.572289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204796.572292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204796.572313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204797.596226:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204797.596235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788204797.596237:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204797.596240:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204797.596244:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204797.596249:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788204797.596250:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204797.596253:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204797.596257:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204797.596260:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788204797.596264:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204797.596265:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204797.596271:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204797.596272:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204797.596277:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204797.596288:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204797.596295:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204797.596298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204797.596303:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204797.596309:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204797.596312:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204797.596317:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204797.596318:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204797.788780:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204797.788785:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204797.789410:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204797.789415:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204797.789620:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204797.789622:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204797.789804:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204797.789806:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1788204797.790047:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204797.790049:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00010000:00000001:0.0:1788204798.620201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204798.620206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204798.620215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204798.620219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204798.620242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204798.620247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204798.620254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204798.620259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204799.132208:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321157893, at 4321157952 00000100:00000010:2.0:1788204799.132226:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00010000:00000001:0.0:1788204799.644185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204799.644191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204799.644198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204799.644201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204799.644205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204799.644209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204799.644220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204799.644223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204799.644274:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204799.644278:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204799.644280:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204800.668185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204800.668189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204800.668194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204800.668196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204800.668208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204800.668210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204800.668216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204800.668219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204800.668280:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204800.668283:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204800.668286:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204801.692172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204801.692179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204801.692187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204801.692191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204801.692201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204801.692205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204801.692224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204801.692226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204802.716207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204802.716211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204802.716220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204802.716222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204802.716227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204802.716231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204802.716234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204802.716237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204802.716288:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204802.716292:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204802.716296:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204802.908228:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204802.908240:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204802.908242:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204802.908245:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204802.908249:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204802.908255:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26489 00000400:00000200:0.0:1788204802.908264:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204802.908269:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204802.908272:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204802.908274:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908276:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908278:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908279:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204802.908282:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204802.908283:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204802.908286:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204802.908289:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204802.908290:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204802.908291:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204802.908293:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908295:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908296:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908297:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204802.908299:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204802.908300:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908302:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204802.908304:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204802.908306:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204802.908308:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908309:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204802.908312:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204802.908319:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204802.908321:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204802.908329:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204802.908330:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908332:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204802.908334:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204802.908338:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204802.908344:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204802.908349:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204802.908357:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543473664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204802.908364:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204802.908367:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204802.908370:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543473664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204802.908385:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204802.908388:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204802.908398:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204802.908399:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204802.908400:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204802.908403:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204802.908406:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26489 00000400:00000200:0.0:1788204802.908410:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204802.908413:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204802.908415:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204802.908417:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908419:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908421:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908423:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204802.908425:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204802.908426:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204802.908428:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204802.908430:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204802.908432:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204802.908434:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204802.908436:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908439:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908440:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908442:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204802.908444:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204802.908447:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908449:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204802.908450:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204802.908455:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204802.908457:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908459:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204802.908462:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204802.908465:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908467:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204802.908470:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204802.908472:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204802.908475:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204802.908476:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908477:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204802.908478:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204802.908480:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204802.908483:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204802.908487:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204802.908491:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543473792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204802.908497:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204802.908499:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204802.908503:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543473792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204802.908512:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204802.908515:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204802.908530:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204802.908532:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204802.908534:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204802.908536:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204802.908538:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26489 00000400:00000200:0.0:1788204802.908564:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204802.908566:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204802.908568:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204802.908570:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908572:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908574:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908576:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204802.908578:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204802.908579:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204802.908581:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204802.908583:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204802.908584:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204802.908586:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204802.908589:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908591:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908592:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908594:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.1:1788204802.908595:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:0.0:1788204802.908596:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204802.908599:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000200:1.1:1788204802.908599:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:0.0:1788204802.908601:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204802.908602:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204802.908604:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204802.908606:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908608:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204802.908609:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204802.908611:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908613:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204802.908616:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204802.908618:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204802.908620:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204802.908621:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908623:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204802.908624:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204802.908626:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204802.908628:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204802.908632:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204802.908636:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543473920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204802.908643:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204802.908644:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204802.908648:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543473920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204802.908655:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204802.908657:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204802.908660:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204802.908662:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204802.908663:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204802.908665:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204802.908668:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26489 00000400:00000200:0.0:1788204802.908671:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204802.908673:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204802.908675:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204802.908677:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908679:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908681:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908683:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204802.908684:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204802.908686:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204802.908687:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204802.908688:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204802.908690:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204802.908692:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204802.908695:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908696:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908698:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908700:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204802.908702:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204802.908705:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908707:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204802.908709:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204802.908711:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204802.908713:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908715:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204802.908717:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204802.908720:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908721:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204802.908724:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204802.908726:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204802.908728:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204802.908730:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908732:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204802.908733:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204802.908735:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204802.908738:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204802.908741:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204802.908750:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543474048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204802.908757:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204802.908760:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204802.908763:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543474048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204802.908771:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204802.908773:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204802.908777:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204802.908778:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204802.908779:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204802.908782:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204802.908784:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26489 00000400:00000200:0.0:1788204802.908787:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204802.908790:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204802.908792:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204802.908794:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908796:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908798:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908800:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204802.908802:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204802.908803:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204802.908804:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204802.908806:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204802.908813:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204802.908815:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204802.908818:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908819:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204802.908821:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908823:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204802.908825:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204802.908828:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204802.908830:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204802.908832:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204802.908834:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204802.908836:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908838:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204802.908839:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204802.908841:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908843:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204802.908846:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204802.908847:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204802.908850:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204802.908851:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204802.908853:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204802.908854:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204802.908855:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204802.908858:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204802.908861:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204802.908865:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543474176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204802.908871:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204802.908872:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204802.908876:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543474176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204802.908883:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204802.908887:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26499) 00000800:00000200:1.1:1788204802.908988:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204802.908990:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204802.909237:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204802.909239:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204802.909404:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204802.909406:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788204802.909641:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204802.909643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00010000:00000001:0.0:1788204803.740194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204803.740200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204803.740216:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204803.740221:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204803.740224:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740228:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204803.740232:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740234:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740237:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204803.740241:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204803.740241:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204803.740242:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740243:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204803.740244:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740245:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740246:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204803.740251:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204803.740251:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204803.740253:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740254:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204803.740255:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740256:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204803.740257:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204803.740296:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204803.740299:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204803.740302:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204804.252217:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321163013, at 4321163072 00000100:00000010:2.0:1788204804.252233:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c00. 00010000:00000001:0.0:1788204804.764207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204804.764212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204804.764218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204804.764220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204804.764226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204804.764228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204804.764234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204804.764237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204804.764304:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204804.764308:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204804.764311:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204805.788211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204805.788219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204805.788227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204805.788231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204805.788239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204805.788243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204805.788249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204805.788253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204805.788310:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204805.788315:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204805.788318:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204806.812189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204806.812195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204806.812201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204806.812205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204806.812210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204806.812214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204806.812222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204806.812226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204806.812278:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204806.812281:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204806.812283:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204807.836208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204807.836219:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204807.836224:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204807.836233:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204807.836244:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204807.836249:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204807.836259:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204807.836272:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204807.836280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204807.836282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204807.836286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204807.836288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204807.836291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204807.836293:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204807.836346:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204807.836350:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204807.836354:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204808.028963:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204808.028978:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204808.029759:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204808.029760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204808.029927:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204808.029928:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204808.030087:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204808.030088:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204808.030329:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204808.030331:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00010000:00000001:0.0:1788204808.860242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204808.860252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204808.860264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204808.860268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204808.860282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204808.860284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204808.860290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204808.860292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204808.860373:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204808.860378:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204808.860382:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204809.372201:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321168134, at 4321168192 00000100:00000010:2.0:1788204809.372218:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00010000:00000001:0.0:1788204809.884218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204809.884227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204809.884256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204809.884259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204809.884264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204809.884267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204809.884273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204809.884275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204809.884325:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204809.884329:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204809.884331:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204810.908200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204810.908210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204810.908226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204810.908230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204810.908238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204810.908243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204810.908253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204810.908264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204810.908337:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204810.908340:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204810.908343:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204811.932195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204811.932201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204811.932207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204811.932209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204811.932214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204811.932216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204811.932221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204811.932222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204811.932285:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204811.932289:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204811.932291:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204812.956219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204812.956231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204812.956246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204812.956249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204812.956267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204812.956272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204812.956277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204812.956279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204812.956348:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204812.956352:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204812.956355:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204813.148756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204813.148760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204813.149363:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204813.149365:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204813.149531:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204813.149533:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204813.149695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204813.149697:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204813.149925:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204813.149926:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00010000:00000001:0.0:1788204813.980184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204813.980190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204813.980202:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204813.980205:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204813.980208:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980211:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204813.980214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980216:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980219:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204813.980223:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204813.980224:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204813.980225:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980227:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204813.980228:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980230:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980232:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204813.980236:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204813.980238:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204813.980239:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204813.980242:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980244:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204813.980245:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204813.980291:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204813.980294:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204813.980297:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204814.492182:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321173253, at 4321173312 00000100:00000010:2.0:1788204814.492199:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00010000:00000001:0.0:1788204815.004176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204815.004187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204815.004194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204815.004196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204815.004200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204815.004203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204815.004209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204815.004212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204815.004261:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204815.004264:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204815.004267:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204816.028191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204816.028194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204816.028202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204816.028205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204816.028210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204816.028213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204816.028217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204816.028220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204816.028286:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204816.028290:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204816.028293:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204817.052185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204817.052190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204817.052194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204817.052197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204817.052203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204817.052206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204817.052214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204817.052218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204817.052288:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204817.052291:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204817.052294:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204818.076188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204818.076193:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204818.076197:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204818.076201:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204818.076207:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204818.076210:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204818.076213:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204818.076216:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204818.076221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204818.076224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204818.076230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204818.076233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204818.076235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204818.076238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204818.076301:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204818.076305:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204818.076308:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:1.1:1788204818.268701:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204818.268706:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204818.269476:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204818.269478:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204818.269671:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204818.269672:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204818.269877:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204818.269879:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204818.270152:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204818.270153:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204819.100198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204819.100202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204819.100207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204819.100210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204819.100214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204819.100216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204819.100219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204819.100222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204819.100372:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204819.100376:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204819.100379:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204819.612188:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321178373, at 4321178432 00000100:00000010:2.0:1788204819.612205:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00010000:00000001:0.0:1788204820.124183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204820.124188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204820.124198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204820.124203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204820.124212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204820.124216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204820.124226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204820.124229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204820.124292:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204820.124296:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204820.124299:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204821.148190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204821.148196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204821.148206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204821.148209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204821.148216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204821.148220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204821.148238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204821.148240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204821.148282:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204821.148285:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204821.148287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204822.172203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204822.172209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204822.172216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204822.172220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204822.172228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204822.172233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204822.172241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204822.172246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204822.172295:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204822.172298:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204822.172300:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204823.196185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204823.196193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204823.196201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204823.196205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204823.196212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204823.196216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204823.196222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204823.196228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204823.196334:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204823.196337:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204823.196340:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204823.388192:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204823.388203:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204823.388205:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204823.388209:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204823.388214:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204823.388221:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26509 00000400:00000200:2.0:1788204823.388233:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204823.388239:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204823.388243:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204823.388246:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388249:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388251:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388254:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204823.388258:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204823.388260:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204823.388263:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204823.388266:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204823.388269:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204823.388271:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204823.388274:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388276:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388278:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388280:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204823.388283:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204823.388286:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388289:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204823.388292:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204823.388296:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204823.388298:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388301:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204823.388303:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204823.388311:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204823.388315:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204823.388318:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204823.388320:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388323:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204823.388325:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204823.388330:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204823.388338:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204823.388343:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204823.388352:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543476224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204823.388360:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204823.388363:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204823.388366:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543476224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204823.388383:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204823.388385:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204823.388388:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204823.388389:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204823.388390:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204823.388392:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204823.388394:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26509 00000400:00000200:2.0:1788204823.388397:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204823.388399:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204823.388401:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204823.388403:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388405:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388407:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388408:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204823.388410:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204823.388411:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204823.388413:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204823.388414:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204823.388415:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204823.388417:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204823.388421:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388424:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388428:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388431:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204823.388433:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204823.388437:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388442:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204823.388445:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204823.388451:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204823.388456:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388458:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204823.388463:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204823.388468:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388474:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204823.388480:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204823.388485:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204823.388491:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204823.388494:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388497:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204823.388499:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204823.388503:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204823.388509:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204823.388513:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204823.388520:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543476352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204823.388527:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204823.388529:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204823.388535:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543476352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204823.388546:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204823.388549:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204823.388553:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204823.388556:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204823.388558:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204823.388562:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204823.388566:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26509 00000400:00000200:2.0:1788204823.388570:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204823.388572:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204823.388574:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204823.388575:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388577:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388587:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388589:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204823.388590:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204823.388592:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204823.388594:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204823.388596:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204823.388598:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204823.388602:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204823.388607:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388609:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388610:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388611:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204823.388615:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204823.388619:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388620:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204823.388623:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204823.388627:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204823.388630:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388633:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204823.388636:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204823.388640:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388645:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204823.388650:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204823.388654:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204823.388659:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204823.388660:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388662:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204823.388663:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204823.388665:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204823.388670:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204823.388675:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204823.388680:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543476480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204823.388688:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204823.388690:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204823.388695:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543476480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204823.388706:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204823.388710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204823.388711:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204823.388715:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204823.388716:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1788204823.388716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00080000:2.0:1788204823.388718:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204823.388723:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204823.388729:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26509 00000400:00000200:2.0:1788204823.388736:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204823.388741:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204823.388745:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204823.388752:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388756:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388759:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388764:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204823.388767:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204823.388769:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204823.388773:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204823.388776:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204823.388781:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204823.388785:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204823.388792:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388795:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388798:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388802:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204823.388806:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204823.388813:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388819:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204823.388822:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204823.388826:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204823.388843:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388846:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204823.388848:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204823.388851:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388855:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204823.388859:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204823.388863:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204823.388867:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204823.388869:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.388873:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204823.388876:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204823.388878:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204823.388883:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204823.388888:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204823.388896:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543476608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204823.388910:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204823.388914:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204823.388921:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543476608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204823.388939:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204823.388941:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204823.388945:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204823.388948:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204823.388950:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204823.388955:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204823.388960:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26509 00000400:00000200:2.0:1788204823.388968:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204823.388975:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204823.388980:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204823.388987:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388990:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.388994:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.388999:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204823.389003:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204823.389005:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204823.389009:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204823.389011:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204823.389015:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204823.389021:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204823.389026:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.389030:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204823.389034:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204823.389037:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204823.389041:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204823.389045:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204823.389049:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204823.389053:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204823.389056:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204823.389059:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.389061:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204823.389063:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204823.389066:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.389069:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204823.389073:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204823.389077:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204823.389080:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204823.389083:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204823.389085:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204823.389088:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204823.389110:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204823.389114:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204823.389116:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204823.389120:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543476736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204823.389128:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204823.389131:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204823.389135:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543476736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204823.389142:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204823.389148:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26519) 00000800:00000200:1.1:1788204823.389302:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204823.389304:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204823.389471:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204823.389472:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204823.389664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204823.389666:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788204823.389866:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204823.389868:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00010000:00000001:0.0:1788204824.220189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204824.220194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204824.220211:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204824.220213:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204824.220219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220227:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204824.220231:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204824.220250:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204824.220251:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204824.220252:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220253:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204824.220254:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220255:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220256:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204824.220260:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204824.220261:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204824.220263:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220264:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204824.220265:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220266:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204824.220268:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204824.220315:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204824.220318:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204824.220320:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204824.732181:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321183493, at 4321183552 00000100:00000010:2.0:1788204824.732195:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. 00010000:00000001:0.0:1788204825.244183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204825.244188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204825.244195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204825.244197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204825.244201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204825.244203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204825.244208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204825.244212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204825.244261:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204825.244265:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204825.244268:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204826.268178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204826.268182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204826.268189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204826.268193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204826.268196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204826.268200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204826.268203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204826.268206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204826.268252:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204826.268255:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204826.268257:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204827.292198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204827.292204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204827.292213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204827.292217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204827.292222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204827.292226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204827.292234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204827.292238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204827.292299:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204827.292303:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204827.292306:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204828.316186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204828.316192:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204828.316196:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204828.316199:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204828.316204:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204828.316208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204828.316213:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204828.316217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204828.316222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204828.316225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204828.316231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204828.316233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204828.316237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204828.316238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204828.316292:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204828.316295:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204828.316297:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204828.508181:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204828.508189:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204828.508191:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204828.508194:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204828.508197:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204828.508202:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26514 00000400:00000200:2.0:1788204828.508210:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204828.508215:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204828.508217:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204828.508219:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508221:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508223:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508224:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204828.508227:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204828.508228:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204828.508230:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204828.508233:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204828.508234:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204828.508236:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204828.508238:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508239:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508240:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508241:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204828.508243:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204828.508245:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508247:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204828.508249:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204828.508251:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204828.508253:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508255:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204828.508256:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204828.508264:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204828.508274:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204828.508277:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204828.508278:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508280:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204828.508282:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204828.508285:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204828.508291:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204828.508296:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204828.508305:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543476864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204828.508314:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204828.508319:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204828.508323:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543476864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204828.508345:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204828.508352:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204828.508354:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204828.508355:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204828.508356:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204828.508358:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204828.508360:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26514 00000400:00000200:2.0:1788204828.508363:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204828.508365:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204828.508367:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204828.508370:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508373:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508375:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508378:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204828.508381:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204828.508383:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204828.508384:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204828.508387:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204828.508390:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204828.508394:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204828.508401:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508406:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508411:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508415:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204828.508418:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204828.508422:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508425:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204828.508429:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204828.508435:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204828.508438:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508442:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204828.508447:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204828.508451:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508453:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204828.508457:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204828.508460:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204828.508462:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204828.508464:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508468:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204828.508469:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204828.508471:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204828.508477:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204828.508484:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204828.508492:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543476992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204828.508508:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204828.508513:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204828.508522:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543476992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204828.508535:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204828.508539:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204828.508545:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204828.508547:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204828.508548:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204828.508550:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204828.508554:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26514 00000400:00000200:2.0:1788204828.508560:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204828.508562:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204828.508563:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204828.508568:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508572:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508574:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508575:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204828.508577:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204828.508578:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204828.508580:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204828.508582:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204828.508585:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204828.508587:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204828.508591:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508594:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508599:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508603:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204828.508606:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204828.508611:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508616:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204828.508620:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204828.508624:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204828.508628:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508630:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204828.508631:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204828.508633:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508635:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204828.508641:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204828.508645:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204828.508648:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204828.508649:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508652:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204828.508654:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204828.508655:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204828.508658:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204828.508662:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204828.508669:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543477120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204828.508681:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1788204828.508681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204828.508685:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204828.508686:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000100:00000040:2.0:1788204828.508691:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543477120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204828.508703:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204828.508705:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204828.508711:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204828.508714:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204828.508716:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204828.508718:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204828.508721:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26514 00000400:00000200:2.0:1788204828.508727:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204828.508731:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204828.508733:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204828.508738:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508754:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508759:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508763:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204828.508768:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204828.508771:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204828.508775:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204828.508779:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204828.508783:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204828.508788:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204828.508793:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508796:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508799:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508802:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204828.508807:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204828.508809:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508811:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204828.508814:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204828.508816:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204828.508818:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508822:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204828.508824:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204828.508827:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508832:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204828.508839:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204828.508845:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204828.508851:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204828.508853:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.508857:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204828.508861:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204828.508864:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204828.508870:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204828.508877:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204828.508884:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543477248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204828.508902:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204828.508905:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204828.508910:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543477248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204828.508936:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204828.508940:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204828.508945:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204828.508947:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204828.508950:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204828.508955:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204828.508960:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26514 00000400:00000200:2.0:1788204828.508966:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204828.508971:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204828.508973:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204828.508978:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508980:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.508984:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.508987:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204828.508991:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204828.508993:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204828.508994:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204828.508997:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204828.509000:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204828.509004:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204828.509011:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.509015:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204828.509021:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204828.509026:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204828.509033:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204828.509040:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204828.509046:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204828.509050:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204828.509055:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204828.509058:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.509063:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204828.509066:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204828.509069:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.509071:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204828.509075:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204828.509079:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204828.509084:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204828.509087:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204828.509116:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204828.509119:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204828.509122:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204828.509129:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204828.509134:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204828.509142:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543477376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204828.509154:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204828.509158:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204828.509164:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543477376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204828.509176:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204828.509181:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26525) 00000800:00000200:1.1:1788204828.509243:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204828.509244:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788204828.509413:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204828.509415:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788204828.509590:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204828.509592:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788204828.509861:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204828.509863:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00010000:00000001:0.0:1788204829.340168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204829.340174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204829.340179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204829.340182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204829.340188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204829.340191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204829.340196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204829.340199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204829.340258:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204829.340260:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204829.340262:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204829.852161:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321188613, at 4321188672 00000100:00000010:2.0:1788204829.852172:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00010000:00000001:0.0:1788204830.364181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204830.364187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204830.364195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204830.364199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204830.364204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204830.364207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204830.364213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204830.364216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204830.364263:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204830.364266:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204830.364268:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204831.388186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204831.388190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204831.388198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204831.388201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204831.388208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204831.388212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204831.388218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204831.388221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204831.388334:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204831.388337:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204831.388339:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204832.412187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204832.412193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204832.412204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204832.412208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204832.412216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204832.412220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204832.412229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204832.412232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204832.412278:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204832.412281:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204832.412283:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204833.436249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204833.436258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204833.436270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204833.436274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204833.436280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204833.436287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204833.436293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204833.436300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204833.436374:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204833.436379:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204833.436383:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204833.628217:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204833.628231:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204833.628234:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204833.628239:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204833.628245:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204833.628261:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26520 00000400:00000200:2.0:1788204833.628273:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204833.628280:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204833.628284:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204833.628287:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628291:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.628303:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628307:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204833.628312:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204833.628314:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204833.628317:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204833.628321:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204833.628324:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204833.628326:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204833.628329:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628331:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.628333:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628335:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204833.628338:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204833.628341:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628344:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204833.628347:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204833.628351:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204833.628354:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.628357:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204833.628360:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204833.628372:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204833.628377:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204833.628382:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204833.628383:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.628387:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204833.628391:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204833.628396:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204833.628405:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204833.628413:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204833.628422:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543477504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204833.628432:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204833.628437:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204833.628442:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543477504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204833.628472:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204833.628476:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204833.628479:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204833.628481:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204833.628482:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204833.628485:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204833.628488:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26520 00000400:00000200:2.0:1788204833.628493:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204833.628497:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204833.628500:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204833.628504:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628507:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.628509:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628512:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204833.628514:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204833.628516:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204833.628518:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204833.628522:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204833.628529:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204833.628536:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204833.628544:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628548:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.628555:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628564:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204833.628578:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204833.628593:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628601:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204833.628613:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204833.628623:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204833.628632:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.628638:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204833.628643:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204833.628654:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.628671:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204833.628677:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204833.628683:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204833.628689:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204833.628692:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.628695:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204833.628698:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204833.628702:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204833.628708:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204833.628718:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204833.628734:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543477632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204833.628755:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204833.628759:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204833.628767:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543477632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204833.628787:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204833.628791:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204833.628805:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204833.628810:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204833.628814:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204833.628818:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204833.628824:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26520 00000400:00000200:2.0:1788204833.628837:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204833.628851:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204833.628861:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204833.628873:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628888:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.628900:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628913:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204833.628922:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204833.628925:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204833.628931:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204833.628937:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000800:00000200:1.1:1788204833.628943:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788204833.628945:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000800:00000200:1.1:1788204833.628948:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000020:00000001:2.0:1788204833.628954:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204833.628967:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628972:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.628983:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.628989:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204833.628993:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204833.629012:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629019:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204833.629026:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204833.629031:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204833.629037:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629040:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204833.629045:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204833.629057:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629063:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204833.629073:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204833.629079:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204833.629114:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204833.629117:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629120:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204833.629122:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204833.629130:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204833.629147:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204833.629155:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204833.629169:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543477760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204833.629200:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204833.629208:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204833.629219:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543477760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204833.629253:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204833.629260:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204833.629264:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204833.629266:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204833.629268:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204833.629274:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204833.629284:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26520 00000400:00000200:2.0:1788204833.629294:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204833.629303:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204833.629313:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204833.629323:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629328:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.629336:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629342:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204833.629347:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204833.629351:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204833.629353:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204833.629356:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204833.629360:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204833.629362:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204833.629369:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629372:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.629378:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629380:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204833.629387:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204833.629393:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629396:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204833.629400:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204833.629403:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204833.629409:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629415:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204833.629418:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204833.629422:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629426:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204833.629431:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204833.629438:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204833.629448:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204833.629453:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629460:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204833.629463:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204833.629471:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204833.629483:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204833.629491:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204833.629499:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543477888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204833.629516:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204833.629521:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204833.629532:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543477888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204833.629555:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204833.629563:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204833.629579:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204833.629584:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204833.629588:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204833.629599:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204833.629606:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26520 00000400:00000200:2.0:1788204833.629615:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204833.629628:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204833.629638:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204833.629648:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629653:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.629669:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629672:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204833.629676:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204833.629679:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204833.629689:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204833.629698:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204833.629708:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204833.629712:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204833.629721:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629728:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204833.629737:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629739:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204833.629750:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204833.629754:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204833.629756:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204833.629760:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204833.629766:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204833.629773:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629781:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204833.629785:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204833.629790:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629801:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204833.629811:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204833.629818:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204833.629829:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204833.629837:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204833.629842:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204833.629849:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204833.629855:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204833.629870:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204833.629882:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204833.629896:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543478016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204833.629948:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204833.629960:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204833.629968:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543478016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204833.629983:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204833.629989:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26530) 00000800:00000200:1.1:1788204833.630046:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204833.630048:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204833.630328:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204833.630331:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204833.630565:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204833.630566:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204833.630853:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204833.630854:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00010000:00000001:0.0:1788204834.460187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204834.460193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204834.460210:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204834.460215:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204834.460220:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460226:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204834.460230:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460234:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460260:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204834.460266:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204834.460267:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204834.460269:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460279:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204834.460280:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460282:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460284:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460288:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204834.460290:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204834.460291:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204834.460293:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460294:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204834.460295:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460297:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204834.460299:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204834.460349:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204834.460353:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204834.460356:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204834.972192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321193734, at 4321193792 00000100:00000010:2.0:1788204834.972207:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00010000:00000001:0.0:1788204835.484184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204835.484190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204835.484198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204835.484201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204835.484205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204835.484209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204835.484218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204835.484221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204835.484273:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204835.484277:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204835.484281:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204836.508177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204836.508182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204836.508190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204836.508193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204836.508204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204836.508208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204836.508216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204836.508220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204836.508272:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204836.508276:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204836.508279:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204837.532187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204837.532194:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204837.532199:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204837.532203:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204837.532208:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204837.532217:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204837.532221:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204837.532226:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204837.532233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204837.532235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204837.532239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204837.532240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204837.532243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204837.532245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204837.532296:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204837.532300:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204837.532303:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204838.556223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204838.556229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204838.556239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204838.556244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204838.556253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204838.556256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204838.556261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204838.556263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204838.556320:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204838.556323:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204838.556325:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204838.748259:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204838.748269:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204838.748272:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204838.748275:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204838.748279:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204838.748286:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26525 00000400:00000200:0.0:1788204838.748296:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204838.748302:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204838.748310:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204838.748313:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748315:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.748318:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748319:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204838.748323:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204838.748325:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204838.748328:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204838.748331:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204838.748332:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204838.748334:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204838.748336:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748337:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.748338:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748340:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204838.748342:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204838.748344:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748346:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204838.748348:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204838.748351:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204838.748353:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748355:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204838.748357:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204838.748367:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788204838.748369:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204838.748373:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204838.748374:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748377:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204838.748380:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204838.748384:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204838.748393:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788204838.748399:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204838.748406:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543478144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204838.748413:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204838.748418:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204838.748423:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543478144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204838.748445:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204838.748449:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204838.748452:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204838.748453:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204838.748455:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204838.748458:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204838.748460:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26525 00000400:00000200:0.0:1788204838.748463:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204838.748465:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204838.748467:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204838.748469:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748471:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.748472:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748474:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204838.748475:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204838.748476:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204838.748477:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204838.748479:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204838.748480:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204838.748482:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204838.748484:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748485:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.748487:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748488:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204838.748491:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204838.748495:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748498:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204838.748501:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204838.748508:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204838.748510:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748511:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204838.748515:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204838.748518:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748522:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204838.748527:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000020:00000040:0.0:1788204838.748531:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204838.748534:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204838.748537:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748541:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204838.748544:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204838.748548:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204838.748554:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204838.748560:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204838.748583:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543478272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204838.748601:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204838.748603:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204838.748607:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543478272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204838.748616:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204838.748619:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204838.748623:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204838.748625:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204838.748627:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204838.748630:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204838.748633:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26525 00000400:00000200:0.0:1788204838.748638:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204838.748641:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204838.748643:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204838.748646:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748649:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.748650:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748652:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204838.748655:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204838.748658:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204838.748661:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204838.748666:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204838.748672:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204838.748676:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204838.748682:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748689:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.748693:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748704:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204838.748708:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204838.748712:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748718:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204838.748720:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204838.748722:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204838.748726:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748728:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204838.748731:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204838.748733:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748734:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204838.748737:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788204838.748739:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204838.748743:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204838.748746:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748748:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204838.748750:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204838.748751:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204838.748755:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204838.748759:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204838.748764:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543478400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204838.748774:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204838.748777:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204838.748782:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543478400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204838.748792:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204838.748797:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204838.748803:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204838.748806:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204838.748808:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204838.748811:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204838.748816:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26525 00000400:00000200:0.0:1788204838.748820:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204838.748825:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204838.748833:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204838.748836:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748837:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.748841:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748845:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204838.748848:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204838.748850:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204838.748852:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204838.748854:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204838.748857:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204838.748859:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204838.748865:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748867:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.748868:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748870:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204838.748873:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204838.748876:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204838.748877:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204838.748878:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204838.748880:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204838.748883:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748885:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204838.748887:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204838.748888:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.748889:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204838.748892:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788204838.748893:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204838.748898:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204838.748901:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204838.748902:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788204838.748904:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204838.748913:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204838.748915:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204838.748919:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204838.748922:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204838.748929:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543478528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204838.748942:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204838.748945:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204838.748951:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543478528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204838.748961:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204838.748963:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204838.748966:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204838.748968:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204838.748971:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204838.748974:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204838.748976:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26525 00000400:00000200:0.0:1788204838.748980:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204838.748992:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204838.748997:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204838.749003:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.749008:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.749013:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.749017:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204838.749020:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204838.749022:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204838.749024:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204838.749030:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204838.749034:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204838.749038:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204838.749043:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.749046:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204838.749049:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204838.749051:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204838.749055:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204838.749060:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204838.749065:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204838.749069:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204838.749075:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204838.749079:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.749085:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204838.749088:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204838.749113:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.749115:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204838.749118:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788204838.749120:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204838.749122:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204838.749124:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204838.749128:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204838.749130:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204838.749138:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204838.749145:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204838.749149:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204838.749155:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543478656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204838.749164:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204838.749167:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204838.749172:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543478656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204838.749180:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204838.749186:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26535) 00000800:00000200:1.1:1788204838.749265:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204838.749427:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204838.749580:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204838.749740:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00010000:00000001:0.0:1788204839.580228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204839.580233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204839.580245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204839.580248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204839.580254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204839.580258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204839.580263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204839.580267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204839.580348:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204839.580352:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204839.580356:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204840.092265:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321198853, at 4321198912 00000100:00000010:2.0:1788204840.092271:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00010000:00000001:0.0:1788204840.604329:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204840.604334:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204840.604339:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204840.604340:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204840.604349:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204840.604350:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204840.604354:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204840.604355:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204840.604394:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204840.604397:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204840.604400:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204841.628191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204841.628197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204841.628202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204841.628205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204841.628208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204841.628211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204841.628219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204841.628221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204841.628271:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204841.628275:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204841.628278:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204842.652230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204842.652235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204842.652244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204842.652246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204842.652251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204842.652254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204842.652257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204842.652259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204842.652337:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204842.652342:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204842.652346:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204843.676226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204843.676231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204843.676245:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204843.676248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204843.676254:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676258:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204843.676262:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676265:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676271:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204843.676279:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204843.676279:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204843.676281:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676282:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204843.676283:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676285:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676286:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204843.676291:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204843.676293:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204843.676295:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676296:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204843.676298:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676299:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204843.676301:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204843.676361:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204843.676366:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204843.676370:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204843.868286:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204843.868303:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204843.868307:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204843.868311:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204843.868316:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204843.868325:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26530 00000400:00000200:2.0:1788204843.868336:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204843.868344:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204843.868347:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204843.868351:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868354:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868357:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868360:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204843.868365:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204843.868368:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204843.868371:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204843.868375:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204843.868379:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204843.868381:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204843.868384:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868386:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868388:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868390:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204843.868393:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204843.868396:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868399:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204843.868410:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204843.868415:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204843.868418:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868420:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204843.868423:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204843.868431:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204843.868436:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204843.868441:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204843.868443:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868447:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204843.868451:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204843.868456:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204843.868466:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204843.868474:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204843.868494:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543478784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204843.868505:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204843.868511:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204843.868515:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543478784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204843.868545:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204843.868548:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204843.868552:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204843.868554:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204843.868555:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204843.868558:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204843.868561:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26530 00000400:00000200:2.0:1788204843.868565:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204843.868568:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204843.868570:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204843.868573:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868575:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868577:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868579:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204843.868581:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204843.868583:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204843.868585:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204843.868587:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204843.868589:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204843.868591:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204843.868594:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868596:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868598:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868600:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204843.868602:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204843.868605:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868607:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204843.868609:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204843.868617:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204843.868618:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868620:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204843.868625:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204843.868629:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868631:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204843.868635:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204843.868637:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204843.868639:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204843.868641:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868642:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204843.868644:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204843.868646:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204843.868649:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204843.868653:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204843.868658:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543478912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204843.868665:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204843.868667:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204843.868670:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543478912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204843.868677:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204843.868680:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204843.868684:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204843.868685:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204843.868687:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204843.868690:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204843.868692:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26530 00000400:00000200:2.0:1788204843.868696:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204843.868699:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204843.868701:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204843.868704:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868706:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868708:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868710:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204843.868712:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204843.868714:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204843.868715:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204843.868718:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204843.868719:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204843.868721:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204843.868724:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868726:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868728:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868729:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204843.868732:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204843.868734:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868737:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204843.868739:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204843.868741:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204843.868743:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868745:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204843.868747:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204843.868749:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868751:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204843.868754:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204843.868756:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204843.868759:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204843.868760:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868762:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204843.868763:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204843.868765:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204843.868768:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204843.868771:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204843.868775:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543479040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204843.868787:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204843.868789:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204843.868793:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543479040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204843.868800:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204843.868803:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204843.868806:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204843.868808:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204843.868809:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204843.868812:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204843.868814:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26530 00000400:00000200:2.0:1788204843.868818:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204843.868821:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204843.868823:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204843.868826:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868828:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868830:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868832:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204843.868834:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204843.868835:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204843.868837:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204843.868839:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204843.868841:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204843.868843:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204843.868845:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868847:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868849:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868851:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204843.868853:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204843.868856:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868858:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204843.868859:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204843.868862:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204843.868864:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868866:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204843.868868:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204843.868870:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868872:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204843.868875:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204843.868877:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204843.868880:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204843.868881:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868882:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204843.868883:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204843.868885:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204843.868888:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204843.868891:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204843.868894:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543479168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204843.868901:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204843.868903:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204843.868906:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543479168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204843.868913:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204843.868915:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204843.868918:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204843.868919:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204843.868921:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204843.868924:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204843.868927:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26530 00000400:00000200:2.0:1788204843.868930:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204843.868933:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204843.868935:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204843.868938:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868940:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868942:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868943:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204843.868946:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204843.868947:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204843.868948:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204843.868950:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204843.868952:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204843.868954:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204843.868957:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868959:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204843.868961:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868963:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204843.868965:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204843.868967:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204843.868969:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204843.868971:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204843.868974:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204843.868976:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868978:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204843.868980:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204843.868982:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868983:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204843.868986:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204843.868988:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204843.868990:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204843.868992:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204843.868994:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204843.868995:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204843.868997:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204843.869000:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204843.869003:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204843.869006:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543479296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204843.869012:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204843.869014:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204843.869016:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543479296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204843.869024:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204843.869029:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26540) 00000800:00000200:1.1:1788204843.869388:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204843.869392:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204843.870074:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204843.870077:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204843.870439:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204843.870442:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204843.870744:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204843.870747:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204843.871021:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204843.871023:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00010000:00000001:0.0:1788204844.700230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204844.700236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204844.700244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204844.700246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204844.700251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204844.700253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204844.700261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204844.700263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204844.700381:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204844.700386:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204844.700389:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204845.212241:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321203974, at 4321204032 00000100:00000010:2.0:1788204845.212263:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647000. 00010000:00000001:0.0:1788204845.724227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204845.724233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204845.724241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204845.724245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204845.724252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204845.724255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204845.724262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204845.724265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204845.724324:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204845.724327:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204845.724330:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204846.748216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204846.748221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204846.748227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204846.748229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204846.748233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204846.748236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204846.748240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204846.748242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204846.748313:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204846.748318:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204846.748321:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204847.772223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204847.772232:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204847.772237:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204847.772244:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204847.772250:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204847.772255:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204847.772261:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204847.772268:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204847.772277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204847.772279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204847.772283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204847.772292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204847.772298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204847.772299:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204847.772358:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204847.772362:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204847.772366:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204848.796177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204848.796182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204848.796192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204848.796195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204848.796202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204848.796210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204848.796216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204848.796223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204848.796287:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204848.796290:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204848.796293:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204848.988180:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.988190:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204848.988193:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204848.988197:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204848.988202:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204848.988209:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26535 00000400:00000200:0.0:1788204848.988219:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204848.988227:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204848.988230:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204848.988233:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988236:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988239:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988242:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204848.988247:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204848.988249:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204848.988252:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204848.988256:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204848.988259:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204848.988261:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204848.988265:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988267:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988269:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988271:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204848.988274:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204848.988277:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988280:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204848.988282:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204848.988286:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204848.988289:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988291:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204848.988294:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204848.988303:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204848.988307:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204848.988310:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204848.988312:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988315:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204848.988318:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204848.988322:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204848.988328:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204848.988333:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204848.988341:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543479424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204848.988349:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204848.988352:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204848.988355:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543479424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204848.988378:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204848.988381:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.988383:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204848.988384:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204848.988386:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204848.988387:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204848.988410:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26535 00000400:00000200:0.0:1788204848.988414:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204848.988416:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204848.988417:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204848.988419:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988420:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988422:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988423:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204848.988424:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204848.988425:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204848.988426:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204848.988427:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204848.988428:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204848.988430:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204848.988431:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988432:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988433:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988435:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204848.988436:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204848.988438:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988439:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204848.988440:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204848.988444:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204848.988446:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988447:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204848.988450:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204848.988451:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988452:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204848.988455:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204848.988456:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204848.988458:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204848.988459:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988460:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204848.988461:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204848.988462:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204848.988464:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204848.988467:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204848.988470:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543479552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204848.988475:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204848.988476:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204848.988478:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543479552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204848.988483:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204848.988485:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.988488:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204848.988489:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204848.988490:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204848.988491:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204848.988492:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26535 00000400:00000200:0.0:1788204848.988494:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204848.988496:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204848.988497:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204848.988498:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988499:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988501:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988502:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204848.988503:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204848.988504:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204848.988505:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204848.988506:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204848.988507:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204848.988508:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204848.988510:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988511:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988512:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988513:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204848.988514:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204848.988516:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988517:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204848.988518:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204848.988520:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204848.988521:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988523:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204848.988525:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204848.988527:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988529:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204848.988531:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204848.988533:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204848.988535:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204848.988536:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988537:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204848.988538:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204848.988540:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204848.988543:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204848.988547:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204848.988552:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543479680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204848.988559:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204848.988562:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204848.988569:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543479680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204848.988583:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204848.988590:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.988596:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204848.988600:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204848.988603:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204848.988608:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204848.988614:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26535 00000400:00000200:0.0:1788204848.988623:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204848.988631:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204848.988635:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204848.988640:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988645:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988648:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988650:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204848.988653:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204848.988657:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204848.988661:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204848.988665:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204848.988668:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204848.988671:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204848.988674:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988679:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988680:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988681:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204848.988683:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204848.988685:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988689:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204848.988690:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204848.988693:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204848.988695:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988706:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204848.988711:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204848.988717:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988722:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204848.988728:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204848.988733:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204848.988737:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204848.988740:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988743:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204848.988745:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204848.988748:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204848.988756:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204848.988761:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204848.988769:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543479808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204848.988781:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204848.988785:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204848.988793:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543479808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204848.988803:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204848.988805:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204848.988811:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204848.988814:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204848.988816:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204848.988820:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204848.988823:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26535 00000400:00000200:0.0:1788204848.988826:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204848.988828:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204848.988830:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204848.988835:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988839:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988842:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988847:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204848.988848:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204848.988850:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204848.988852:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204848.988854:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204848.988858:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204848.988860:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204848.988868:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988873:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204848.988879:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204848.988884:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204848.988888:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000800:00000200:1.1:1788204848.988889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204848.988895:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000200:1.1:1788204848.988895:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1788204848.988902:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204848.988906:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204848.988911:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204848.988915:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988917:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204848.988920:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204848.988924:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988927:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204848.988933:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204848.988935:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204848.988939:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204848.988942:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204848.988945:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204848.988947:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204848.988949:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204848.988954:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204848.988960:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204848.988968:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543479936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204848.989003:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204848.989007:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204848.989014:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543479936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204848.989029:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204848.989036:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26545) 00000800:00000200:1.1:1788204848.989309:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204848.989311:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204848.989703:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204848.989705:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204848.990000:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204848.990003:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788204848.990306:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204848.990308:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00010000:00000001:0.0:1788204849.820175:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204849.820179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204849.820186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204849.820190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204849.820197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204849.820200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204849.820208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204849.820212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204849.820267:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204849.820272:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204849.820275:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204850.332191:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321209093, at 4321209152 00000100:00000010:2.0:1788204850.332207:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00010000:00000001:0.0:1788204850.844169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204850.844173:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204850.844181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204850.844185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204850.844193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204850.844197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204850.844201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204850.844202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204850.844252:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204850.844255:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204850.844258:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204851.868179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204851.868185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204851.868194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204851.868197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204851.868204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204851.868208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204851.868214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204851.868218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204851.868262:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204851.868265:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204851.868267:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204852.892191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204852.892194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204852.892199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204852.892202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204852.892208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204852.892210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204852.892217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204852.892220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204852.892277:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204852.892281:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204852.892284:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204853.916171:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204853.916175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204853.916187:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204853.916190:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204853.916193:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916197:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204853.916200:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916205:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916210:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204853.916215:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204853.916217:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204853.916220:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916221:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204853.916222:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916224:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916225:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204853.916228:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204853.916229:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204853.916230:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204853.916232:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916233:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204853.916234:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204853.916283:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204853.916286:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204853.916288:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204854.108164:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204854.108172:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204854.108173:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204854.108176:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204854.108179:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204854.108185:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26540 00000400:00000200:0.0:1788204854.108192:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204854.108198:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204854.108200:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204854.108203:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108205:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108207:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108209:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204854.108211:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204854.108212:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204854.108215:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204854.108217:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204854.108219:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204854.108220:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204854.108222:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108223:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108224:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108225:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204854.108227:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204854.108229:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108230:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204854.108232:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204854.108234:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204854.108236:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108238:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204854.108240:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204854.108246:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788204854.108248:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204854.108251:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204854.108252:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108254:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204854.108256:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204854.108259:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204854.108264:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204854.108269:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204854.108282:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543480064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204854.108290:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204854.108294:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204854.108297:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543480064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204854.108327:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204854.108329:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204854.108331:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204854.108332:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204854.108333:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204854.108335:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204854.108337:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26540 00000400:00000200:0.0:1788204854.108340:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204854.108342:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204854.108343:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204854.108345:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108346:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108348:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108349:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204854.108351:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204854.108352:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204854.108353:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204854.108355:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204854.108356:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204854.108357:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204854.108359:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108361:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108363:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108365:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204854.108368:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204854.108371:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108374:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204854.108376:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204854.108380:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204854.108385:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108388:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204854.108392:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204854.108396:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108399:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204854.108402:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788204854.108406:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204854.108409:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204854.108411:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108414:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204854.108417:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204854.108421:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204854.108424:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788204854.108429:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204854.108436:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543480192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204854.108444:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204854.108448:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204854.108453:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543480192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204854.108460:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204854.108463:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204854.108465:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204854.108469:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204854.108472:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204854.108476:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204854.108480:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26540 00000400:00000200:0.0:1788204854.108485:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204854.108491:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204854.108495:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204854.108500:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108505:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108507:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108510:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204854.108513:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204854.108515:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204854.108516:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204854.108518:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204854.108521:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204854.108523:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204854.108525:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108526:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108527:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108528:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204854.108529:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204854.108531:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108532:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204854.108533:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204854.108534:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204854.108537:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108539:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204854.108541:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204854.108544:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108545:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204854.108550:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788204854.108552:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204854.108555:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204854.108557:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108559:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204854.108562:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204854.108565:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204854.108570:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788204854.108574:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204854.108580:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543480320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204854.108587:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204854.108592:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204854.108599:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543480320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204854.108619:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204854.108624:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788204854.108625:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1788204854.108631:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00080000:0.0:1788204854.108632:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204854.108635:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204854.108638:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204854.108642:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204854.108647:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26540 00000400:00000200:0.0:1788204854.108653:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204854.108656:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204854.108659:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204854.108662:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108665:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108669:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108672:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204854.108681:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204854.108684:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204854.108686:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204854.108688:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204854.108691:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204854.108695:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204854.108701:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108703:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108705:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108708:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204854.108710:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204854.108714:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108719:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204854.108721:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204854.108724:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204854.108728:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108730:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204854.108732:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204854.108735:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108738:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204854.108740:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204854.108741:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204854.108744:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204854.108746:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108749:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204854.108751:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204854.108752:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204854.108756:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788204854.108760:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204854.108765:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543480448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204854.108773:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204854.108775:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204854.108778:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543480448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204854.108787:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204854.108790:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204854.108793:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204854.108797:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204854.108800:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204854.108805:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204854.108808:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26540 00000400:00000200:0.0:1788204854.108813:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204854.108818:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204854.108822:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204854.108825:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108828:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108829:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108831:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204854.108834:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204854.108836:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204854.108838:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204854.108840:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204854.108842:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204854.108845:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204854.108847:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108851:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204854.108854:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108857:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204854.108861:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204854.108865:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204854.108866:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204854.108870:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204854.108872:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204854.108876:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108879:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204854.108882:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204854.108885:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108887:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204854.108891:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204854.108895:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204854.108898:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204854.108900:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204854.108902:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204854.108904:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204854.108908:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204854.108912:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788204854.108915:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204854.108923:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543480576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204854.108931:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204854.108935:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204854.108941:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543480576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204854.108966:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204854.108970:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26550) 00000800:00000200:1.1:1788204854.109138:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204854.109141:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204854.109357:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204854.109359:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204854.109524:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204854.109526:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788204854.109724:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204854.109726:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00010000:00000001:0.0:1788204854.940185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204854.940190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204854.940195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204854.940197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204854.940204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204854.940206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204854.940214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204854.940217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204854.940267:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204854.940270:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204854.940272:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204855.452164:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321214213, at 4321214272 00000100:00000010:2.0:1788204855.452176:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00010000:00000001:0.0:1788204855.964168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204855.964174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204855.964178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204855.964181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204855.964188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204855.964190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204855.964194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204855.964196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204855.964246:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204855.964248:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204855.964251:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204856.988185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204856.988191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204856.988198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204856.988201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204856.988207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204856.988211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204856.988220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204856.988223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204856.988295:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204856.988297:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204856.988300:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204858.012160:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204858.012166:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204858.012170:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204858.012174:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204858.012178:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204858.012181:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204858.012186:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204858.012192:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204858.012198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204858.012201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204858.012204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204858.012206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204858.012213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204858.012214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204859.036170:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204859.036174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204859.036179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204859.036182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204859.036185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204859.036188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204859.036192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204859.036195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204859.036258:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204859.036262:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204859.036266:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204859.228231:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204859.228243:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204859.228245:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204859.228249:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204859.228253:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204859.228260:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26545 00000400:00000200:2.0:1788204859.228271:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204859.228279:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204859.228283:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204859.228286:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228289:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.228291:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228293:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204859.228297:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204859.228299:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204859.228301:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204859.228305:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204859.228307:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204859.228310:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204859.228313:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228314:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.228316:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228318:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204859.228321:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204859.228324:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228327:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204859.228330:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204859.228334:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204859.228337:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228339:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204859.228341:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204859.228350:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204859.228353:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204859.228356:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204859.228358:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228361:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204859.228363:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204859.228367:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204859.228374:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204859.228381:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204859.228389:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543480704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204859.228396:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204859.228400:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204859.228403:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543480704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204859.228420:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204859.228423:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204859.228425:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204859.228426:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204859.228427:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204859.228429:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204859.228431:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26545 00000400:00000200:2.0:1788204859.228440:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204859.228443:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204859.228445:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204859.228447:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228450:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.228452:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228454:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204859.228456:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204859.228458:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204859.228460:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204859.228462:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204859.228464:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204859.228469:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204859.228474:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228478:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.228482:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228486:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204859.228490:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204859.228496:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228501:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204859.228504:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204859.228509:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204859.228513:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228514:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204859.228518:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204859.228521:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228524:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204859.228528:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204859.228531:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204859.228533:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204859.228535:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228539:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204859.228540:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204859.228543:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204859.228547:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204859.228551:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204859.228558:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543480832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204859.228565:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204859.228568:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204859.228573:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543480832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204859.228579:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204859.228581:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204859.228585:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204859.228595:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204859.228597:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204859.228599:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204859.228603:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26545 00000400:00000200:2.0:1788204859.228608:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204859.228613:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204859.228615:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204859.228617:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228618:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.228620:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228624:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204859.228626:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204859.228627:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204859.228628:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204859.228630:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204859.228634:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204859.228637:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204859.228644:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228648:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.228654:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228659:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204859.228665:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204859.228672:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228678:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204859.228683:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204859.228687:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204859.228690:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228694:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204859.228704:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204859.228710:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228716:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204859.228720:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204859.228723:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204859.228727:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204859.228729:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228732:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204859.228733:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204859.228736:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204859.228739:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204859.228744:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204859.228751:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543480960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204859.228757:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204859.228760:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204859.228765:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543480960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204859.228773:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204859.228776:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204859.228784:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204859.228786:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204859.228789:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204859.228795:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204859.228802:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26545 00000400:00000200:2.0:1788204859.228811:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204859.228818:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:1.1:1788204859.228819:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204859.228825:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1788204859.228829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00000040:2.0:1788204859.228832:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228836:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.228842:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228848:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204859.228853:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204859.228855:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204859.228858:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204859.228861:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204859.228866:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204859.228868:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204859.228872:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228876:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.228877:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228879:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204859.228884:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204859.228889:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204859.228894:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204859.228898:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204859.228902:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204859.228906:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228908:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204859.228912:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204859.228917:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228921:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204859.228927:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204859.228931:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204859.228936:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204859.228937:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.228938:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204859.228940:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204859.228941:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204859.228947:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204859.228951:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204859.228956:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543481088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204859.228964:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204859.228967:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204859.228971:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543481088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204859.228979:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204859.228982:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204859.228987:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204859.228990:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204859.228992:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204859.228997:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204859.229002:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26545 00000400:00000200:2.0:1788204859.229012:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204859.229019:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204859.229023:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204859.229030:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.229035:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.229039:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.229044:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204859.229047:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204859.229050:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204859.229053:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204859.229057:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204859.229072:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204859.229078:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204859.229086:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.229128:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204859.229134:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204859.229140:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204859.229145:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204859.229151:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204859.229154:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204859.229158:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204859.229162:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204859.229167:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.229170:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204859.229172:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204859.229176:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.229180:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204859.229184:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788204859.229189:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204859.229192:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204859.229193:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204859.229196:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204859.229199:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204859.229204:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204859.229211:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204859.229231:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204859.229241:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543481216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204859.229255:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204859.229260:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204859.229270:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543481216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204859.229293:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204859.229301:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26555) 00000800:00000200:1.1:1788204859.229399:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204859.229401:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204859.229629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204859.229630:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204859.229827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204859.229829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788204859.230050:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204859.230052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00010000:00000001:0.0:1788204860.060165:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204860.060170:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204860.060177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204860.060180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204860.060183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204860.060186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204860.060193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204860.060205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204860.060257:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204860.060260:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204860.060262:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204860.572205:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321219333, at 4321219392 00000100:00000010:2.0:1788204860.572223:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00010000:00000001:0.0:1788204861.084199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204861.084205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204861.084211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204861.084215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204861.084221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204861.084226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204861.084235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204861.084237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204862.108155:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204862.108159:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204862.108174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204862.108176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204862.108178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204862.108179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204862.108182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204862.108183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204862.108204:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204862.108207:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204862.108211:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204863.132242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204863.132246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204863.132253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204863.132259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204863.132267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204863.132274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204863.132282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204863.132287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204863.132371:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204863.132375:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204863.132379:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204864.156221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204864.156227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204864.156245:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204864.156248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204864.156254:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204864.156265:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156282:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204864.156296:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204864.156298:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204864.156301:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156303:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204864.156305:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156308:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156310:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156314:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204864.156315:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204864.156316:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204864.156318:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156319:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204864.156321:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156322:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204864.156324:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204864.349172:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204864.349182:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1788204864.349359:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204864.349364:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204864.349367:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204864.349373:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26550 00000400:00000200:3.0:1788204864.349378:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204864.349381:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204864.349382:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204864.349384:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204864.349386:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204864.349388:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204864.349389:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204864.349391:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204864.349393:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204864.349395:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204864.349397:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788204864.349400:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788204864.349402:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788204864.349409:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204864.349413:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204864.349418:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204864.349420:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204864.349422:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788204864.349428:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204864.349430:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204864.349432:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204864.349434:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204864.349435:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204864.349437:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204864.349439:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788204864.349440:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204864.349441:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204864.349444:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788204864.349446:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204864.349447:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204864.349448:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204864.349450:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204864.349452:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204864.349453:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204864.349457:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788204864.349459:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204864.349461:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543481856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204864.349474:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204864.349477:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204864.349480:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543481856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204864.349491:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204864.349496:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26560) 00000800:00000200:1.1:1788204864.349824:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204864.349826:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788204864.350152:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204864.350155:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788204864.350434:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204864.350437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788204864.350713:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204864.350716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00010000:00000001:0.0:1788204865.180201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204865.180208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204865.180218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204865.180218:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204865.180223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204865.180226:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204865.180230:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204865.180231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204865.180237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204865.180240:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204865.180243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204865.180251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204865.180251:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204865.180258:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204865.180264:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204865.180270:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788204865.180278:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000100:00000001:3.0:1788204865.180278:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204865.180284:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204865.180289:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204865.692227:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321224454, at 4321224512 00000100:00000040:3.0:1788204865.692227:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321224454, at 4321224512 00000100:00000010:3.0:1788204865.692245:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00000100:00000010:2.0:1788204865.692247:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647960. 00000100:00000001:3.0:1788204866.204217:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204866.204224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204866.204227:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204866.204232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204866.204235:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204866.204243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204866.204245:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204866.204249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204866.204258:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204866.204261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204866.204262:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204866.204270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204866.204276:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204866.204279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204866.204280:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204866.204284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204866.204285:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204866.204354:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204866.204359:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204866.204363:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204867.228226:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204867.228236:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204867.228240:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204867.228249:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204867.228254:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204867.228260:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204867.228266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204867.228270:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204867.228277:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204867.228385:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204867.228388:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204867.228391:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204867.228392:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204867.228394:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204867.228395:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204867.228397:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204867.228398:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204867.228455:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204867.228457:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204867.228460:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204868.252168:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204868.252176:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204868.252180:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204868.252187:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204868.252192:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204868.252197:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204868.252203:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204868.252206:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204868.252210:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204868.252280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204868.252285:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204868.252287:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204868.252290:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204868.252293:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204868.252294:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204868.252296:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204868.252297:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204868.252301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204868.252302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204868.252304:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204868.252305:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204868.252307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204868.252308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204868.252324:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204868.252326:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204868.252328:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204869.276216:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204869.276218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204869.276224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.276226:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204869.276230:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204869.276231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204869.276233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204869.276236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204869.276236:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204869.276238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204869.276244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204869.276244:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204869.276247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.276249:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204869.276253:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204869.276256:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204869.276259:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204869.276323:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204869.276327:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204869.276331:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204869.468215:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204869.468230:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204869.468233:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204869.468237:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204869.468243:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204869.468252:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26555 00000400:00000200:3.0:1788204869.468263:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204869.468271:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204869.468274:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204869.468277:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468280:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.468282:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468285:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204869.468289:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204869.468291:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788204869.468294:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204869.468298:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788204869.468300:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788204869.468303:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788204869.468306:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468307:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.468309:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468311:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204869.468314:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788204869.468317:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468320:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204869.468323:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204869.468327:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204869.468330:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.468332:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204869.468335:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204869.468348:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788204869.468352:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788204869.468356:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204869.468358:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.468362:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204869.468365:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204869.468370:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204869.468380:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:3.0:1788204869.468388:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204869.468397:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543481984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204869.468408:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204869.468413:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204869.468418:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543481984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204869.468445:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204869.468449:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204869.468452:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204869.468453:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204869.468455:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204869.468458:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204869.468461:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26555 00000400:00000200:3.0:1788204869.468465:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204869.468469:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204869.468472:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204869.468476:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468478:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.468481:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468483:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204869.468486:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204869.468487:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788204869.468490:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204869.468495:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788204869.468498:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788204869.468504:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204869.468512:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468517:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.468521:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468526:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204869.468531:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788204869.468539:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468544:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204869.468548:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204869.468558:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204869.468563:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.468566:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204869.468573:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204869.468578:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.468580:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204869.468587:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788204869.468593:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204869.468598:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204869.468601:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.468604:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204869.468605:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204869.468607:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204869.468614:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788204869.468620:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204869.468629:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543482112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204869.468650:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204869.468655:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204869.468665:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543482112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204869.468680:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204869.468685:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204869.468689:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788204869.468692:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204869.468695:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204869.468700:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204869.468707:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26555 00000400:00000200:3.0:1788204869.468716:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204869.468724:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204869.468729:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204869.468735:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468741:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.468747:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468752:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204869.468757:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204869.468759:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788204869.468761:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204869.468764:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788204869.468768:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788204869.468773:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788204869.468781:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468785:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.468790:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468793:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204869.468799:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788204869.468802:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468808:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204869.468812:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204869.468816:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204869.468821:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.468825:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.1:1788204869.468825:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000008:00100000:3.0:1788204869.468828:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.1:1788204869.468831:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000008:00000001:3.0:1788204869.468833:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.468837:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204869.468843:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788204869.468847:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204869.468851:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204869.468854:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.468858:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204869.468863:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204869.468867:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204869.468873:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788204869.468880:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204869.468889:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543482240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204869.468904:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204869.468909:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204869.468918:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543482240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204869.468930:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204869.468934:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204869.468941:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204869.468944:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204869.468946:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204869.468950:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204869.468954:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26555 00000400:00000200:3.0:1788204869.468960:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204869.468965:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204869.468969:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204869.468972:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.468976:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.469055:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.469062:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204869.469073:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204869.469077:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788204869.469081:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204869.469086:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788204869.469136:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788204869.469142:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788204869.469157:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.469169:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.469177:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.469183:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204869.469196:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788204869.469205:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204869.469209:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204869.469215:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204869.469225:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204869.469229:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.469233:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204869.469238:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788204869.469241:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.469245:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204869.469256:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788204869.469267:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204869.469275:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204869.469278:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.469282:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204869.469288:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204869.469300:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204869.469316:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788204869.469330:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204869.469347:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543482368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204869.469433:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204869.469439:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204869.469447:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543482368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204869.469478:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204869.469482:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204869.469491:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204869.469494:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204869.469500:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204869.469508:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204869.469513:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26555 00000400:00000200:3.0:1788204869.469526:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204869.469535:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204869.469541:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204869.469550:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.469555:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.469564:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.469571:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204869.469579:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204869.469581:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204869.469586:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204869.469591:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788204869.469600:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788204869.469606:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788204869.469619:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204869.469624:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204869.469629:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.1:1788204869.469634:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788204869.469635:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.1:1788204869.469642:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000020:00000040:3.0:1788204869.469644:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788204869.469652:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204869.469662:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204869.469670:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204869.469678:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204869.469684:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.469689:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204869.469693:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788204869.469704:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.469707:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204869.469711:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788204869.469715:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204869.469722:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204869.469725:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204869.469732:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204869.469738:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204869.469742:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204869.469749:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788204869.469754:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204869.469763:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543482496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204869.469795:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204869.469801:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204869.469808:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543482496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204869.469822:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204869.469827:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26565) 00000800:00000200:1.1:1788204869.470087:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204869.470109:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788204869.470458:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204869.470460:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788204869.470686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204869.470688:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000100:00000001:3.0:1788204870.300222:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204870.300232:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204870.300235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204870.300236:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204870.300241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204870.300245:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204870.300249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204870.300252:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204870.300253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204870.300258:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204870.300260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204870.300264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204870.300266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204870.300269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204870.300271:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204870.300272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204870.300279:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204870.300318:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204870.300321:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204870.300323:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204870.812239:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321229573, at 4321229632 00000100:00000040:2.0:1788204870.812241:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321229573, at 4321229632 00000100:00000010:3.0:1788204870.812259:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00000100:00000010:2.0:1788204870.812262:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647660. 00000100:00000001:3.0:1788204871.324231:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204871.324237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204871.324241:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204871.324243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204871.324246:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204871.324252:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204871.324254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204871.324259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204871.324259:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204871.324267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204871.324270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204871.324275:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204871.324277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204871.324281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204871.324283:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204871.324287:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204871.324290:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204871.324310:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204871.324314:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204871.324318:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204872.348227:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204872.348231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204872.348234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204872.348236:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204872.348239:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204872.348240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204872.348244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204872.348244:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204872.348251:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204872.348254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204872.348255:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204872.348260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204872.348266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204872.348271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204872.348276:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204872.348277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204872.348287:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204872.348349:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204872.348353:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204872.348356:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204873.372191:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204873.372194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204873.372200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204873.372203:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204873.372211:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204873.372212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204873.372215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204873.372226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204873.372226:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204873.372233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204873.372239:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204873.372243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204873.372244:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204873.372247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204873.372250:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204873.372258:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204873.372270:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204873.372319:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204873.372324:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204873.372328:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204874.396194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204874.396194:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204874.396201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204874.396203:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204874.396207:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204874.396213:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204874.396214:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204874.396220:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204874.396221:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204874.396225:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204874.396228:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204874.396232:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788204874.396235:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204874.396236:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396241:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204874.396242:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204874.396249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204874.396250:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204874.396261:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204874.396263:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204874.396264:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396266:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204874.396267:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396268:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396270:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204874.396275:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204874.396276:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204874.396277:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396278:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204874.396278:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396280:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204874.396281:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204874.396315:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204874.396319:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204874.396322:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204874.588226:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204874.588238:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204874.588240:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204874.588244:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204874.588249:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204874.588258:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26560 00000400:00000200:2.0:1788204874.588270:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204874.588277:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204874.588281:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204874.588285:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588288:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588290:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588293:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204874.588298:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204874.588301:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204874.588304:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204874.588308:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204874.588310:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204874.588313:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204874.588316:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588318:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588320:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588322:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204874.588325:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204874.588329:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588332:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204874.588335:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204874.588338:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204874.588342:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588345:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204874.588348:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204874.588360:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204874.588365:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204874.588370:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204874.588371:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588374:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204874.588377:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204874.588382:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204874.588391:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204874.588398:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204874.588405:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543482624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204874.588413:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204874.588417:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204874.588420:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543482624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204874.588446:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204874.588448:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204874.588451:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204874.588452:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204874.588453:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204874.588455:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204874.588458:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26560 00000400:00000200:2.0:1788204874.588461:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204874.588463:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204874.588465:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204874.588468:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588469:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588471:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588473:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204874.588475:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204874.588475:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204874.588476:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204874.588478:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204874.588480:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204874.588481:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204874.588483:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588487:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588489:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588491:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204874.588494:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204874.588498:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588501:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204874.588504:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204874.588511:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204874.588514:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588517:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204874.588521:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204874.588524:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588526:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204874.588531:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204874.588532:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204874.588537:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204874.588539:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588541:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204874.588544:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204874.588546:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204874.588550:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204874.588555:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204874.588561:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543482752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204874.588572:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204874.588575:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204874.588577:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543482752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204874.588584:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204874.588587:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204874.588591:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204874.588591:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204874.588592:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204874.588596:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204874.588599:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26560 00000400:00000200:2.0:1788204874.588603:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204874.588605:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204874.588609:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204874.588613:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588614:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588615:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588617:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204874.588622:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204874.588623:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204874.588624:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204874.588626:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204874.588627:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204874.588629:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204874.588633:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588636:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588640:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588642:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204874.588644:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204874.588648:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588652:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204874.588654:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204874.588657:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204874.588660:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588664:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204874.588666:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204874.588669:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588672:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204874.588678:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204874.588682:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204874.588687:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204874.588690:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588694:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204874.588697:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204874.588699:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204874.588706:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204874.588712:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204874.588718:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543482880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204874.588725:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204874.588727:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204874.588733:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543482880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204874.588742:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204874.588746:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204874.588750:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204874.588753:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204874.588755:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204874.588758:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204874.588762:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26560 00000400:00000200:2.0:1788204874.588769:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204874.588773:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204874.588775:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204874.588777:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588779:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588783:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588784:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204874.588785:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204874.588786:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204874.588789:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204874.588792:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204874.588797:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204874.588800:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204874.588806:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588807:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588812:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588815:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204874.588819:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204874.588821:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588823:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204874.588827:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204874.588830:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204874.588831:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588835:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204874.588837:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204874.588840:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588841:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204874.588846:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000800:00000200:1.1:1788204874.588848:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:2.0:1788204874.588849:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000800:00000200:1.1:1788204874.588852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 02000000:00000001:2.0:1788204874.588853:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204874.588855:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.588857:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204874.588860:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204874.588862:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204874.588867:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204874.588870:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204874.588875:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543483008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204874.588885:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204874.588889:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204874.588897:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543483008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204874.588938:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204874.588941:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204874.588944:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204874.588945:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204874.588956:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204874.588961:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204874.588965:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26560 00000400:00000200:2.0:1788204874.588972:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204874.588978:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204874.588980:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204874.588982:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588987:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.588988:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.588990:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204874.588992:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204874.588995:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204874.588997:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204874.588998:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204874.589000:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204874.589005:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204874.589008:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.589011:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204874.589015:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204874.589017:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204874.589021:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204874.589024:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204874.589027:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204874.589030:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204874.589032:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204874.589035:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.589039:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204874.589041:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204874.589046:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.589048:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204874.589053:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204874.589057:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204874.589061:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204874.589064:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204874.589066:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204874.589069:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204874.589070:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204874.589080:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204874.589085:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204874.589122:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543483136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204874.589150:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204874.589152:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204874.589157:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543483136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204874.589166:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204874.589172:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26571) 00000800:00000200:1.1:1788204874.589379:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204874.589381:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204874.589625:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204874.589626:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204874.589843:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204874.589844:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788204874.590119:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204874.590121:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1788204875.420209:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204875.420214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204875.420218:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204875.420220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204875.420222:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204875.420230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204875.420230:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204875.420234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204875.420237:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204875.420241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204875.420243:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204875.420244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204875.420250:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204875.420255:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204875.420260:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204875.420261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204875.420268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204875.420342:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204875.420347:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204875.420349:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204875.932184:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321234693, at 4321234752 00000100:00000040:3.0:1788204875.932184:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321234693, at 4321234752 00000100:00000010:2.0:1788204875.932197:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00000100:00000010:3.0:1788204875.932200:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647900. 00010000:00000001:0.0:1788204876.444221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204876.444222:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204876.444227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204876.444232:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204876.444236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204876.444238:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204876.444243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204876.444252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204876.444252:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204876.444254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204876.444258:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204876.444261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204876.444264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204876.444265:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204876.444281:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204876.444288:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204876.444293:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204876.444333:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204876.444338:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204876.444341:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204877.468151:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204877.468158:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204877.468160:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204877.468166:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204877.468170:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204877.468175:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204877.468180:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204877.468182:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204877.468186:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204877.468253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204877.468257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204877.468262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204877.468264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204877.468267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204877.468268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204877.468271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204877.468272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204877.468291:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204877.468294:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204877.468296:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204878.492194:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204878.492204:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204878.492206:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204878.492208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204878.492214:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:3.0:1788204878.492215:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204878.492218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204878.492220:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204878.492222:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.492226:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204878.492227:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204878.492230:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.492232:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204878.492235:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.492237:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204878.492239:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204878.492243:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204878.492247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204878.492251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204878.492256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204878.492257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204878.492262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204878.492264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204878.492290:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204878.492293:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204878.492296:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204879.516164:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204879.516173:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204879.516176:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204879.516181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204879.516185:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204879.516186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204879.516192:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204879.516196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204879.516196:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204879.516201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204879.516203:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204879.516208:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204879.516210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204879.516213:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204879.516216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204879.516225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204879.516228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204879.516294:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204879.516299:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204879.516302:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204879.708205:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204879.708215:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204879.708218:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204879.708222:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204879.708226:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204879.708241:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26566 00000400:00000200:2.0:1788204879.708251:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204879.708257:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204879.708263:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204879.708266:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708268:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.708270:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708272:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204879.708276:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204879.708278:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204879.708281:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204879.708284:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204879.708286:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204879.708287:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204879.708289:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708291:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.708292:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708293:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204879.708296:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204879.708299:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708300:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204879.708303:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204879.708305:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204879.708308:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.708309:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204879.708312:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204879.708321:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204879.708324:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204879.708328:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204879.708330:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.708333:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204879.708335:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204879.708340:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204879.708348:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204879.708354:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204879.708362:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543483264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204879.708370:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204879.708374:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204879.708378:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543483264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204879.708399:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204879.708402:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204879.708405:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204879.708406:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204879.708407:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204879.708409:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204879.708412:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26566 00000400:00000200:2.0:1788204879.708414:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204879.708416:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204879.708418:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204879.708420:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708421:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.708423:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708425:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204879.708426:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204879.708427:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204879.708428:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204879.708430:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204879.708432:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204879.708435:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204879.708438:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708440:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.708441:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708443:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204879.708445:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204879.708450:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708453:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204879.708456:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204879.708462:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204879.708466:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.708470:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204879.708475:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204879.708479:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.708482:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204879.708488:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204879.708491:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204879.708493:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204879.708495:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.708498:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204879.708500:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204879.708502:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204879.708507:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204879.708511:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204879.708518:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543483392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204879.708528:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204879.708530:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204879.708537:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543483392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204879.708569:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204879.708577:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204879.708585:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204879.708588:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204879.708591:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204879.708598:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204879.708605:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26566 00000400:00000200:2.0:1788204879.708617:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204879.708625:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204879.708632:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204879.708639:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708643:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.708648:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708650:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204879.708651:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204879.708652:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204879.708656:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204879.708659:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204879.708662:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204879.708666:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204879.708672:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708674:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.708685:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708690:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204879.708694:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204879.708701:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708707:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204879.708710:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204879.708713:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204879.708720:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.708726:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204879.708730:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204879.708735:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.708739:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204879.708747:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204879.708754:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000800:00000200:1.1:1788204879.708756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:2.0:1788204879.708760:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1788204879.708760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 02000000:00000001:2.0:1788204879.708764:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.708770:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204879.708775:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204879.708780:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204879.708789:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204879.708796:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204879.708805:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543483520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204879.708817:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204879.708822:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204879.708832:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543483520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204879.708871:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204879.708875:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204879.708878:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204879.708881:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204879.708884:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204879.708890:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204879.708898:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26566 00000400:00000200:2.0:1788204879.708907:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204879.708912:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204879.708917:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204879.708923:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708929:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.708936:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708943:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204879.708948:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204879.708951:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204879.708954:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204879.708960:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204879.708965:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204879.708971:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204879.708981:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708986:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.708991:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.708995:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204879.709001:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204879.709008:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204879.709013:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204879.709019:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204879.709023:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204879.709028:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.709034:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204879.709039:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204879.709044:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.709048:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204879.709055:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204879.709058:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204879.709064:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204879.709066:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.709070:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204879.709072:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204879.709075:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204879.709081:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204879.709089:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204879.709125:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543483648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204879.709141:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204879.709145:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204879.709154:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543483648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204879.709169:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204879.709174:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204879.709183:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204879.709186:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204879.709189:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204879.709193:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204879.709197:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26566 00000400:00000200:2.0:1788204879.709203:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204879.709209:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204879.709213:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204879.709219:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.709224:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.709226:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.709230:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204879.709233:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204879.709236:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204879.709238:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204879.709241:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204879.709245:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204879.709249:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788204879.709252:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.709253:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204879.709256:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204879.709259:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204879.709262:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204879.709267:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204879.709272:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204879.709275:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204879.709279:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204879.709284:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.709287:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204879.709290:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204879.709294:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.709296:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204879.709303:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204879.709307:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204879.709313:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204879.709315:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204879.709318:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204879.709321:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204879.709324:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204879.709331:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204879.709337:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204879.709345:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543483776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204879.709358:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204879.709362:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204879.709366:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543483776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204879.709382:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204879.709388:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26576) 00000800:00000200:1.1:1788204879.709440:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204879.709442:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204879.709666:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204879.709668:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204879.709854:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204879.709856:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788204879.710151:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204879.710153:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1788204880.540215:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204880.540221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204880.540225:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204880.540228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204880.540230:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204880.540237:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204880.540240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204880.540243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204880.540243:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204880.540249:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204880.540251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204880.540254:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204880.540255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204880.540259:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204880.540261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204880.540265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204880.540271:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204880.540299:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204880.540303:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204880.540306:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204881.052227:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321239813, at 4321239872 00000100:00000040:3.0:1788204881.052227:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321239813, at 4321239872 00000100:00000010:3.0:1788204881.052252:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16474e0. 00000100:00000010:2.0:1788204881.052254:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00010000:00000001:0.0:1788204881.564240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204881.564244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204881.564252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204881.564253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204881.564257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204881.564258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204881.564262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204881.564263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204881.564350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204881.564361:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204881.564366:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204881.564375:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204881.564380:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204881.564381:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204881.564384:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204881.564386:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204881.564388:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204881.564390:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204881.564392:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204881.564393:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204882.588199:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204882.588205:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204882.588207:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204882.588212:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204882.588216:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204882.588222:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204882.588225:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204882.588228:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204882.588231:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204882.588240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204882.588247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204882.588256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204882.588257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204882.588260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204882.588262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204882.588265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204882.588267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204882.588309:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204882.588312:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204882.588315:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204883.612183:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204883.612187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204883.612191:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204883.612192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.612196:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204883.612201:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204883.612204:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788204883.612207:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204883.612208:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204883.612212:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204883.612213:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.612218:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204883.612219:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788204883.612222:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204883.612223:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204883.612227:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612228:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612231:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204883.612236:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204883.612239:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204883.612242:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612244:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204883.612246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612248:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612251:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204883.612257:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204883.612258:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204883.612259:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612260:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204883.612261:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612262:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204883.612263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204883.612280:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204883.612282:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204883.612284:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204884.636186:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204884.636189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204884.636192:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204884.636193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.636195:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204884.636199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204884.636201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204884.636202:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204884.636205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204884.636207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.636208:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204884.636210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204884.636212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.636214:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204884.636220:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204884.636224:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204884.636230:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204884.636262:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204884.636267:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204884.636269:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204884.828218:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204884.828228:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204884.828231:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204884.828234:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204884.828238:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204884.828246:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26571 00000400:00000200:3.0:1788204884.828256:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204884.828264:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204884.828267:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204884.828270:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828273:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.828276:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828279:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204884.828283:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204884.828285:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788204884.828289:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204884.828292:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788204884.828294:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788204884.828296:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788204884.828298:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828300:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.828301:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828302:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204884.828304:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788204884.828307:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828308:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204884.828310:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204884.828312:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204884.828314:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828315:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204884.828318:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204884.828325:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788204884.828327:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788204884.828331:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204884.828332:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828334:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204884.828336:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204884.828339:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204884.828345:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412400. 00000100:00000001:3.0:1788204884.828352:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204884.828361:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543483904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204884.828371:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204884.828376:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204884.828380:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543483904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204884.828405:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204884.828409:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204884.828412:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204884.828413:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204884.828415:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204884.828417:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204884.828419:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26571 00000400:00000200:3.0:1788204884.828422:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204884.828424:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204884.828426:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204884.828429:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828431:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.828433:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828434:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204884.828436:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204884.828437:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788204884.828439:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204884.828441:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788204884.828445:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788204884.828450:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204884.828456:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828461:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.828466:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828471:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204884.828477:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788204884.828482:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828484:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204884.828486:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204884.828491:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204884.828495:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828500:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204884.828504:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204884.828515:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828518:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204884.828523:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788204884.828526:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204884.828530:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204884.828532:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828536:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204884.828537:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204884.828540:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204884.828548:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413800. 00000100:00000001:3.0:1788204884.828555:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204884.828565:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543484032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204884.828577:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204884.828581:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204884.828588:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543484032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204884.828604:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204884.828609:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204884.828618:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788204884.828621:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204884.828623:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204884.828629:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204884.828635:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26571 00000400:00000200:3.0:1788204884.828643:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204884.828651:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204884.828655:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204884.828662:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828668:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.828674:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828677:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204884.828681:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204884.828683:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788204884.828686:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204884.828689:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788204884.828692:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788204884.828696:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788204884.828702:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828707:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.828710:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828712:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204884.828715:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788204884.828720:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828722:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204884.828726:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204884.828730:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204884.828732:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828735:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204884.828737:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204884.828740:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828741:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204884.828744:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788204884.828755:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204884.828758:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204884.828761:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828764:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204884.828765:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204884.828768:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204884.828769:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204884.828788:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 02000000:00000010:3.0:1788204884.828792:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:3.0:1788204884.828797:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204884.828804:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543484160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204884.828811:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204884.828815:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204884.828821:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543484160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204884.828827:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204884.828833:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204884.828839:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204884.828840:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204884.828842:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204884.828846:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204884.828850:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26571 00000400:00000200:3.0:1788204884.828854:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204884.828859:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204884.828863:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204884.828868:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828870:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.828874:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828876:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204884.828877:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204884.828881:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788204884.828882:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204884.828885:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788204884.828887:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788204884.828890:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788204884.828896:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828899:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.828903:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828906:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204884.828911:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788204884.828915:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204884.828917:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204884.828918:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204884.828920:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204884.828922:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828925:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204884.828926:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788204884.828929:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828932:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204884.828934:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788204884.828939:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204884.828940:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204884.828941:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.828942:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204884.828943:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204884.828945:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204884.828952:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788204884.828958:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204884.828965:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543484288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204884.828980:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204884.828985:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204884.828992:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543484288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204884.829018:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204884.829023:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204884.829030:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204884.829034:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204884.829037:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204884.829042:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204884.829048:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26571 00000400:00000200:3.0:1788204884.829054:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204884.829061:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204884.829066:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204884.829071:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.829075:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.829079:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.829083:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204884.829086:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204884.829125:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204884.829128:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204884.829130:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788204884.829133:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788204884.829138:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788204884.829146:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.829150:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204884.829155:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204884.829161:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204884.829166:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788204884.829172:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204884.829177:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204884.829180:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204884.829185:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204884.829189:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.829194:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204884.829199:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788204884.829205:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.829208:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204884.829212:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788204884.829215:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204884.829221:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204884.829224:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204884.829228:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204884.829231:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204884.829233:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204884.829239:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788204884.829245:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204884.829254:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543484416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204884.829265:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204884.829269:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204884.829275:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543484416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204884.829289:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204884.829294:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26581) 00000800:00000200:1.1:1788204884.829452:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204884.829455:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204884.829728:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204884.829730:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204884.829967:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204884.829968:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204884.830138:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204884.830140:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00010000:00000001:0.0:1788204885.660212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204885.660219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204885.660224:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204885.660227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204885.660238:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204885.660239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204885.660243:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204885.660248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204885.660253:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204885.660254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204885.660260:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204885.660267:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204885.660270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204885.660271:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204885.660277:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204885.660278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204885.660281:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204885.660342:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204885.660345:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204885.660347:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204886.172237:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321244933, at 4321244992 00000100:00000040:3.0:1788204886.172237:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321244933, at 4321244992 00000100:00000010:2.0:1788204886.172257:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c00. 00000100:00000010:3.0:1788204886.172260:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00000100:00000001:3.0:1788204886.684246:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204886.684249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204886.684255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204886.684258:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204886.684264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204886.684264:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204886.684268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204886.684277:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204886.684279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204886.684285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204886.684286:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204886.684291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204886.684292:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204886.684299:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204886.684300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204886.684303:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204886.684309:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204886.684342:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204886.684345:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204886.684347:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204887.708236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204887.708243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204887.708257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204887.708262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204887.708270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204887.708274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204887.708280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204887.708281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204887.708347:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204887.708351:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204887.708354:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204887.708418:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204887.708424:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204887.708425:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204887.708429:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204887.708433:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204887.708436:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204887.708439:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204887.708442:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204887.708445:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.732267:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204888.732268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204888.732281:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788204888.732283:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204888.732285:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204888.732295:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204888.732296:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204888.732305:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:3.0:1788204888.732305:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204888.732312:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.732314:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204888.732321:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.732323:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204888.732333:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.732334:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204888.732342:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204888.732345:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204888.732353:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204888.732358:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204888.732359:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204888.732367:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204888.732377:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204888.732380:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204888.732476:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204888.732481:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204888.732486:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204889.756185:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204889.756192:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204889.756198:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204889.756202:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204889.756207:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204889.756211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204889.756212:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.756217:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204889.756218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.756222:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.756227:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204889.756228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204889.756233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204889.756247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204889.756249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204889.756253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204889.756255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204889.756304:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204889.756308:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204889.756313:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204889.948517:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204889.948525:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204889.948532:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204889.948544:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.948565:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204889.948568:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.948576:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204889.948582:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204889.948588:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204889.948599:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543484544:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788204889.948619:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204889.948627:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204889.948633:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.948639:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204889.948649:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204889.948655:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204889.948664:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204889.948673:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204889.948698:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204889.948708:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543484544, portal 25 00000100:00000001:3.0:1788204889.948712:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204889.948716:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788204889.948729:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.948751:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204889.948760:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204889.948766:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204889.948773:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543484544, offset 0 00000400:00000200:3.0:1788204889.948785:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204889.948799:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.948813:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204889.948824:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204889.948833:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204889.948841:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204889.948848:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204889.948855:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471134:471134:256:4294967295] 192.168.201.118@tcp LPNI seq info [470993:470993:8:4294967295] 00000400:00000200:3.0:1788204889.948866:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.948883:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204889.948894:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.948903:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204889.948917:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204889.948925:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204889.948940:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204889.948946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204889.948956:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.948962:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204889.948974:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204889.948997:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204889.949008:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204889.949014:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204889.949024:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204889.949035:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949054:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.949063:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949083:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.949113:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204889.949123:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204889.949130:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204889.949138:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321250054 00000100:00000010:3.0:1788204889.949149:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fce40. 00000100:00000001:3.0:1788204889.949160:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788204889.949171:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321250054 00000100:00000010:3.0:1788204889.949179:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc840. 00000100:00000001:3.0:1788204889.949186:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788204889.949194:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204889.949199:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.949215:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204889.949230:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204889.949253:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204889.949257:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204889.949262:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204889.949269:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204889.949274:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204889.949280:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204889.949289:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204889.949311:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949332:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204889.949336:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949341:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.949345:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204889.949354:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949370:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204889.949379:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949403:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204889.949412:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949418:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.949422:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949427:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204889.949431:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204889.949439:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.949450:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204889.949456:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204889.949464:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.949474:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543484544:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788204889.949481:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949484:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.949488:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543484544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949496:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204889.949499:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204889.949503:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204889.949507:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204889.949509:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204889.949511:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204889.949514:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204889.949517:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:3.0:1788204889.949522:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204889.949524:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204889.949526:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788204889.949532:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204889.949534:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949536:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949545:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204889.949547:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.949550:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204889.949551:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204889.949553:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204889.949557:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949566:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204889.949567:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.949570:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204889.949572:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204889.949575:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204889.949578:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543484672:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204889.949582:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204889.949584:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204889.949586:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.949588:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204889.949592:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204889.949594:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204889.949599:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204889.949602:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204889.949607:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204889.949610:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543484672, portal 10 00000100:00000001:3.0:1788204889.949613:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204889.949614:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000040:3.0:1788204889.949623:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949630:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204889.949633:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204889.949636:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204889.949638:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543484672, offset 0 00000400:00000200:3.0:1788204889.949643:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204889.949648:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.949652:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204889.949655:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204889.949659:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204889.949662:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204889.949664:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204889.949668:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471135:471135:256:4294967295] 192.168.201.118@tcp LPNI seq info [470994:470994:8:4294967295] 00000400:00000200:3.0:1788204889.949672:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.949679:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204889.949702:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.949706:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204889.949711:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204889.949714:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204889.949720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204889.949723:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204889.949728:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.949730:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204889.949733:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204889.949737:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204889.949741:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204889.949743:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204889.949746:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204889.949750:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949758:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.949762:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949769:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.949771:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204889.949774:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204889.949779:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204889.949783:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204889.949785:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321248787 00000100:00000001:3.0:1788204889.949788:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321250054 00000100:00000001:3.0:1788204889.949791:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321248787 00000100:00000001:3.0:1788204889.949793:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321250054 00000100:00000001:3.0:1788204889.949795:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204889.949798:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.949800:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204889.949804:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204889.949813:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204889.949815:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204889.949817:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204889.949820:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204889.949821:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204889.949824:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204889.949827:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204889.949837:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949844:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204889.949846:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949848:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.949849:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204889.949854:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949861:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204889.949865:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949873:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204889.949876:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949877:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.949879:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.949881:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204889.949883:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204889.949886:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.949889:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204889.949892:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204889.949897:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.949906:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543484672:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204889.949911:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204889.949914:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949921:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204889.949923:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.949926:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204889.949928:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204889.949930:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204889.949933:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543484800:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204889.949937:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204889.949939:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204889.949941:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.949943:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204889.949947:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788204889.949949:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204889.949953:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204889.949956:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204889.949960:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204889.949963:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543484800, portal 10 00000100:00000001:3.0:1788204889.949965:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204889.949966:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788204889.949973:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.949980:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204889.949983:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204889.949985:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204889.949988:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543484800, offset 0 00000400:00000200:3.0:1788204889.949993:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204889.949997:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.950003:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204889.950007:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204889.950009:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204889.950012:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204889.950015:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204889.950018:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471136:471136:256:4294967295] 192.168.201.118@tcp LPNI seq info [470995:470995:8:4294967295] 00000400:00000200:3.0:1788204889.950022:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.950029:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204889.950034:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.950037:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204889.950042:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204889.950045:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204889.950049:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204889.950052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204889.950056:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.950059:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204889.950062:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204889.950066:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204889.950069:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204889.950071:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204889.950074:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204889.950078:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950087:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.950109:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950118:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.950119:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204889.950123:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204889.950127:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204889.950130:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204889.950133:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321248789 00000100:00000001:3.0:1788204889.950136:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321250054 00000100:00000001:3.0:1788204889.950138:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321248789 00000100:00000001:3.0:1788204889.950140:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321250054 00000100:00000001:3.0:1788204889.950143:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204889.950145:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.950148:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204889.950152:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204889.950162:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204889.950163:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204889.950165:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204889.950168:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204889.950170:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204889.950172:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204889.950176:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204889.950185:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950192:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204889.950194:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950196:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.950198:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204889.950202:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950210:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204889.950214:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950221:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204889.950224:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950226:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.950227:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950229:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204889.950232:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204889.950235:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.950238:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204889.950241:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204889.950245:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.950254:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543484800:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204889.950258:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204889.950262:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950269:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204889.950270:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.950273:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204889.950275:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204889.950277:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204889.950280:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543484928:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204889.950283:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204889.950285:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204889.950287:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.950288:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204889.950292:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788204889.950294:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204889.950298:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204889.950301:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204889.950305:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204889.950309:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543484928, portal 4 00000100:00000001:3.0:1788204889.950312:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204889.950313:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788204889.950320:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950327:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204889.950330:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204889.950333:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204889.950335:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543484928, offset 0 00000400:00000200:3.0:1788204889.950339:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204889.950344:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.950348:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204889.950351:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204889.950354:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204889.950357:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204889.950359:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204889.950363:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471137:471137:256:4294967295] 192.168.201.118@tcp LPNI seq info [470996:470996:8:4294967295] 00000400:00000200:3.0:1788204889.950367:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.950374:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204889.950379:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.950382:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204889.950386:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204889.950389:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204889.950393:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204889.950396:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204889.950400:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.950403:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204889.950406:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204889.950409:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204889.950412:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204889.950414:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204889.950417:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204889.950421:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950436:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.950440:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950448:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.950449:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204889.950453:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204889.950456:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204889.950459:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204889.950461:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321248798 00000100:00000001:3.0:1788204889.950464:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321250054 00000100:00000001:3.0:1788204889.950467:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321248798 00000100:00000001:3.0:1788204889.950469:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321250054 00000100:00000001:3.0:1788204889.950472:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204889.950474:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.950476:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204889.950480:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204889.950489:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204889.950491:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204889.950492:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204889.950495:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204889.950497:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204889.950499:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204889.950503:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204889.950512:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950518:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204889.950520:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950522:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.950524:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204889.950527:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950534:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204889.950538:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950545:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204889.950548:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950549:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.950551:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950553:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204889.950555:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204889.950557:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.950560:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204889.950563:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204889.950567:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.950575:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543484928:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204889.950579:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204889.950582:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950589:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204889.950591:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.950593:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204889.950595:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204889.950597:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204889.950601:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485056:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204889.950605:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204889.950606:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204889.950609:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204889.950611:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204889.950614:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788204889.950617:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204889.950620:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204889.950623:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204889.950627:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204889.950630:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543485056, portal 4 00000100:00000001:3.0:1788204889.950632:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204889.950634:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788204889.950640:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950647:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204889.950649:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204889.950652:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204889.950654:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543485056, offset 0 00000400:00000200:3.0:1788204889.950659:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204889.950663:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.950667:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204889.950670:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204889.950673:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204889.950675:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204889.950678:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204889.950694:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471138:471138:256:4294967295] 192.168.201.118@tcp LPNI seq info [470997:470997:8:4294967295] 00000400:00000200:3.0:1788204889.950699:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204889.950706:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204889.950711:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.950715:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204889.950720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204889.950723:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788204889.950727:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204889.950730:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788204889.950735:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204889.950737:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204889.950740:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204889.950744:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204889.950747:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204889.950749:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204889.950752:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204889.950757:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950766:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.950769:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204914 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950777:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.950778:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204889.950782:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204889.950785:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204889.950788:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204889.950791:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321248821 00000100:00000001:3.0:1788204889.950794:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321250054 00000100:00000001:3.0:1788204889.950796:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321248821 00000100:00000001:3.0:1788204889.950798:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321250054 00000100:00000001:3.0:1788204889.950801:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204889.950803:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.950805:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204889.950809:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204889/real 1788204889] req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204889.950819:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204889.950821:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204889.950822:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204889.950825:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204889.950827:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204889.950829:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204889.950833:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204889.950842:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950849:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204889.950850:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950852:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.950853:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204889.950857:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950864:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204889.950868:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950875:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204889.950879:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950880:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204889.950882:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950884:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204889.950886:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204889.950889:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204889.950892:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204889.950895:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204889.950899:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204889.950913:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485056:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204889.950917:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950919:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.950923:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543484672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950930:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204889.950932:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204889.950934:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204889.950937:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204889.950939:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204889.950941:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204889.950943:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204889.950945:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:3.0:1788204889.950947:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204889.950949:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204889.950951:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788204889.950958:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204889.950959:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950961:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.950965:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543484800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.950973:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204889.950974:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204889.950976:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788204889.950979:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204889.950980:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204889.950981:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204889.950984:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204889.950985:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788204889.950988:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204889.950989:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204889.950991:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788204889.950994:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204889.950996:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.950997:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.951001:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543484928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.951008:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204889.951009:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204889.951011:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788204889.951013:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204889.951014:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204889.951016:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204889.951018:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204889.951019:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:3.0:1788204889.951021:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204889.951023:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204889.951024:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788204889.951027:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204889.951028:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.951030:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204889.951033:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543485056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204914 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204889.951041:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204889.951042:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204889.951044:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788204889.951046:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204889.951047:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204889.951048:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204889.951050:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204889.951052:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:3.0:1788204889.951054:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204889.951055:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204889.951056:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788204889.951059:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204889.951060:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.951062:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204889.951066:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204889.951068:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204889.951070:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204889.951072:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204890.780210:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204890.780222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204890.780222:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204890.780225:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204890.780232:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204890.780237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204890.780238:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204890.780244:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204890.780248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204890.780250:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204890.780253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204890.780255:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204890.780259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204890.780263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204890.780263:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204890.780272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204890.780275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204890.780322:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204890.780328:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204890.780333:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204891.292222:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321250054, at 4321250112 00000100:00000040:3.0:1788204891.292222:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321250054, at 4321250112 00000100:00000010:2.0:1788204891.292240:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fce40. 00000100:00000010:3.0:1788204891.292252:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc840. 00000100:00000001:3.0:1788204891.804214:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204891.804221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204891.804221:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204891.804226:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204891.804227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204891.804234:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204891.804236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204891.804240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204891.804242:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204891.804245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204891.804247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204891.804248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204891.804255:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204891.804256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204891.804260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204891.804260:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204891.804268:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204891.804320:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204891.804324:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204891.804326:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204892.828218:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204892.828222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204892.828230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204892.828230:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204892.828235:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204892.828246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204892.828249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204892.828250:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204892.828257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204892.828260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204892.828260:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204892.828267:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204892.828269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204892.828274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204892.828275:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204892.828281:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204892.828288:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204892.828347:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204892.828351:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204892.828354:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204893.852206:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204893.852210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204893.852215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.852216:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204893.852222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204893.852222:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852227:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:3.0:1788204893.852230:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204893.852231:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204893.852236:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.852238:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204893.852241:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204893.852243:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.852244:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204893.852246:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.852250:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204893.852252:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204893.852254:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204893.852256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204893.852258:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852259:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204893.852263:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204893.852266:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852270:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204893.852272:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852276:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204893.852281:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204893.852282:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204893.852283:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852284:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204893.852285:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852287:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204893.852289:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204893.852350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204893.852354:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204893.852357:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204894.876173:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204894.876186:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204894.876188:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204894.876194:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204894.876199:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204894.876201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204894.876204:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204894.876205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204894.876213:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204894.876214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204894.876217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204894.876220:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204894.876223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204894.876226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204894.876227:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204894.876230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204894.876234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204894.876261:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204894.876264:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204894.876267:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204895.068249:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204895.068261:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204895.068264:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204895.068268:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204895.068273:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204895.068282:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26581 00000400:00000200:2.0:1788204895.068294:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204895.068302:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204895.068307:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204895.068311:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068315:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204895.068318:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068320:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204895.068326:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204895.068329:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204895.068333:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204895.068337:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204895.068340:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204895.068344:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204895.068347:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068350:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204895.068361:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068362:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204895.068365:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204895.068368:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068370:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204895.068373:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204895.068377:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204895.068379:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068381:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204895.068384:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204895.068395:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204895.068399:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204895.068403:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204895.068405:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068407:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204895.068410:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204895.068415:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204895.068424:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204895.068431:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204895.068439:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204895.068448:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204895.068452:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204895.068456:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204895.068476:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204895.068479:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204895.068481:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204895.068483:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204895.068484:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204895.068486:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204895.068488:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26581 00000400:00000200:2.0:1788204895.068491:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204895.068494:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204895.068496:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204895.068498:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068499:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204895.068501:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068503:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204895.068505:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204895.068506:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204895.068507:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204895.068510:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204895.068512:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204895.068514:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204895.068516:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204895.068517:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204895.068519:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204895.068523:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204895.068524:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204895.068525:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204895.068529:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:3.0:1788204895.068529:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788204895.068534:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068537:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204895.068538:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:3.0:1788204895.068540:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204895.068544:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204895.068549:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068552:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204895.068555:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1788204895.068557:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788204895.068558:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788204895.068563:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.068565:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788204895.068566:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.068569:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1788204895.068570:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 02000000:00000001:3.0:1788204895.068573:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204895.068574:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204895.068578:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788204895.068579:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543485184:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788204895.068580:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068584:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204895.068587:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204895.068587:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204895.068589:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.068592:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1788204895.068606:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204895.068611:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.068611:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.068616:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788204895.068619:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:3.0:1788204895.068623:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:2.0:1788204895.068628:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788204895.068629:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068633:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.068638:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788204895.068639:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788204895.068644:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204895.068654:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000001:2.0:1788204895.068655:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204895.068660:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204895.068663:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204895.068665:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:3.0:1788204895.068665:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543485184, portal 25 00000100:00080000:2.0:1788204895.068667:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204895.068670:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204895.068671:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204895.068675:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331915776 : -109424377635840 : ffff9c7aa6d75c00) 00000100:00080000:2.0:1788204895.068676:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26581 00000400:00000200:2.0:1788204895.068682:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788204895.068686:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204895.068688:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204895.068694:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204895.068701:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204895.068703:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788204895.068707:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204895.068708:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204895.068713:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000040:2.0:1788204895.068714:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000200:3.0:1788204895.068716:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543485184, offset 0 00000100:00000001:2.0:1788204895.068720:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204895.068724:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204895.068725:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204895.068729:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204895.068732:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204895.068736:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788204895.068737:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000020:00000001:2.0:1788204895.068743:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204895.068748:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204895.068749:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204895.068754:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204895.068757:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068760:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204895.068760:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000040:2.0:1788204895.068764:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788204895.068769:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1788204895.068770:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068774:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.068776:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000001:2.0:1788204895.068778:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204895.068782:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788204895.068782:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1788204895.068786:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.068788:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068789:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204895.068792:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788204895.068795:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471139:471139:256:4294967295] 192.168.201.118@tcp LPNI seq info [470998:470998:8:4294967295] 00000008:00000001:2.0:1788204895.068796:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068799:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.068803:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:2.0:1788204895.068804:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204895.068807:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204895.068810:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204895.068812:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068816:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204895.068818:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204895.068818:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:2.0:1788204895.068821:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204895.068825:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788204895.068828:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204895.068832:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204895.068838:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204895.068840:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204895.068842:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788204895.068853:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068854:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204895.068860:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204895.068863:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000010:3.0:1788204895.068863:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000100:00000001:2.0:1788204895.068866:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204895.068869:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204895.068872:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.0:1788204895.068874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204895.068877:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26581 00000800:00000200:3.0:1788204895.068879:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000400:00000200:2.0:1788204895.068884:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.1:1788204895.068889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:2.0:1788204895.068890:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204895.068895:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1788204895.068897:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000040:2.0:1788204895.068900:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068904:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1788204895.068905:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000040:2.0:1788204895.068908:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068911:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000010:3.0:1788204895.068911:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000020:00000001:2.0:1788204895.068922:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000200:3.0:1788204895.068923:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:2.0:1788204895.068924:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204895.068927:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204895.068930:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204895.068932:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204895.068935:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788204895.068937:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1788204895.068940:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068943:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204895.068944:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.068945:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204895.068947:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:3.0:1788204895.068947:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000100:00080000:2.0:1788204895.068952:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:3.0:1788204895.068954:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:2.0:1788204895.068955:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204895.068959:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204895.068961:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204895.068963:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068965:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204895.068967:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204895.068969:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068971:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.068971:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:2.0:1788204895.068977:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000100:00000200:3.0:1788204895.068979:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204895.068980:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204895.068986:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204895.068988:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.068992:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204895.068994:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204895.068996:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.068998:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1788204895.069000:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000040:3.0:1788204895.069004:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204895.069005:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204895.069014:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204895.069019:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.069025:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204895.069025:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204895.069028:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069032:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204895.069033:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204895.069042:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204895.069044:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069047:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321255173 00000100:00080000:2.0:1788204895.069048:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204895.069053:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204895.069053:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204895.069055:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:3.0:1788204895.069056:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc840. 00000100:00000001:2.0:1788204895.069061:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204895.069064:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26581 00000400:00000200:2.0:1788204895.069067:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204895.069067:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000400:00000200:2.0:1788204895.069073:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204895.069077:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321255173 00000100:00080000:2.0:1788204895.069080:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000010:3.0:1788204895.069081:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc360. 00000100:00000040:2.0:1788204895.069085:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204895.069087:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1788204895.069088:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204895.069114:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.069117:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204895.069119:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204895.069120:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204895.069121:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204895.069122:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204895.069125:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204895.069127:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788204895.069127:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788204895.069134:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204895.069135:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.069138:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204895.069140:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1788204895.069143:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204895.069146:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00100000:3.0:1788204895.069148:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204895/real 1788204895] req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204895.069149:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204895.069155:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204895.069160:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204895.069164:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204895.069169:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1788204895.069171:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788204895.069173:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788204895.069176:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788204895.069177:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:3.0:1788204895.069181:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000008:00100000:2.0:1788204895.069182:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204895.069188:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.069189:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000100:00000001:2.0:1788204895.069191:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204895.069194:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000010:2.0:1788204895.069198:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000100:00000001:3.0:1788204895.069199:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1788204895.069203:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204895.069209:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000200:3.0:1788204895.069209:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204895.069212:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204895.069216:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204895.069219:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204895.069222:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204895.069226:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204895.069227:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204895.069232:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069238:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:2.0:1788204895.069241:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204895.069242:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069247:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.069250:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204895.069256:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204895.069259:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204895.069260:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204895.069265:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069282:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204895.069285:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204895.069289:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204895.069291:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26591) 00000100:00000001:3.0:1788204895.069298:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204895.069301:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069303:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.069304:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069306:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204895.069308:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204895.069310:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204895.069313:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204895.069315:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204895.069318:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204895.069323:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543485184:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788204895.069326:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069328:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.069330:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75c00 x1875055543485184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069334:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204895.069336:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204895.069338:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788204895.069341:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204895.069342:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204895.069343:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204895.069345:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204895.069347:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e800. 02000000:00000001:3.0:1788204895.069358:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204895.069361:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204895.069362:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75c00. 00000100:00000001:3.0:1788204895.069365:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204895.069366:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069368:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069375:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204895.069376:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069378:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204895.069379:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204895.069381:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204895.069383:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069388:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204895.069389:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.069391:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204895.069392:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204895.069394:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204895.069396:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204895.069398:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204895.069399:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204895.069401:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.069402:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204895.069404:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788204895.069406:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.069408:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204895.069410:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204895.069419:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204895.069421:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543485312, portal 10 00000100:00000001:3.0:1788204895.069422:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204895.069423:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331918464 : -109424377633152 : ffff9c7aa6d76680) 00000100:00000040:3.0:1788204895.069429:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069434:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204895.069435:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204895.069437:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204895.069439:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543485312, offset 0 00000400:00000200:3.0:1788204895.069442:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204895.069445:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204895.069448:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204895.069450:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204895.069452:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204895.069454:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204895.069455:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204895.069458:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471140:471140:256:4294967295] 192.168.201.118@tcp LPNI seq info [470999:470999:8:4294967295] 00000400:00000200:3.0:1788204895.069460:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204895.069465:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204895.069468:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204895.069471:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204895.069473:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204895.069476:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204895.069479:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204895.069481:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788204895.069483:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204895.069485:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204895.069487:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204895.069489:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204895.069491:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204895.069493:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204895.069495:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204895.069497:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069502:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.069504:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069509:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069510:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204895.069513:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204895.069517:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204895.069519:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204895.069521:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321253907 00000100:00000001:3.0:1788204895.069523:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321255173 00000100:00000001:3.0:1788204895.069525:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321253907 00000100:00000001:3.0:1788204895.069527:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321255173 00000100:00000001:3.0:1788204895.069528:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204895.069530:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069531:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204895.069533:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204895/real 1788204895] req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204895.069538:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204895.069539:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204895.069539:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204895.069541:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204895.069542:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204895.069543:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204895.069545:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204895.069551:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069555:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204895.069556:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069557:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.069558:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204895.069560:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069564:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204895.069566:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069571:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204895.069573:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069574:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.069575:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069576:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204895.069578:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204895.069579:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204895.069581:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204895.069582:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204895.069585:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204895.069589:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204895.069592:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204895.069608:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069614:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204895.069615:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.069616:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204895.069617:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204895.069619:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204895.069620:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204895.069623:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204895.069624:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204895.069625:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.069627:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204895.069629:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788204895.069631:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.069633:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204895.069635:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204895.069637:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204895.069639:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543485440, portal 10 00000100:00000001:3.0:1788204895.069641:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204895.069642:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331914880 : -109424377636736 : ffff9c7aa6d75880) 00000100:00000040:3.0:1788204895.069646:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069650:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204895.069652:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204895.069654:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204895.069655:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543485440, offset 0 00000400:00000200:3.0:1788204895.069658:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204895.069660:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204895.069662:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204895.069664:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204895.069666:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204895.069667:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204895.069669:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204895.069671:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471141:471141:256:4294967295] 192.168.201.118@tcp LPNI seq info [471000:471000:8:4294967295] 00000400:00000200:3.0:1788204895.069674:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204895.069678:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204895.069681:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204895.069683:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204895.069686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204895.069688:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204895.069691:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204895.069692:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788204895.069695:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204895.069696:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204895.069698:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204895.069701:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204895.069703:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204895.069705:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204895.069706:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204895.069709:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069714:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.069716:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069720:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069721:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204895.069723:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204895.069725:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204895.069727:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204895.069728:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321253908 00000100:00000001:3.0:1788204895.069730:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321255173 00000100:00000001:3.0:1788204895.069731:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321253908 00000100:00000001:3.0:1788204895.069733:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321255173 00000100:00000001:3.0:1788204895.069734:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204895.069736:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069737:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204895.069739:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204895/real 1788204895] req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204895.069744:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204895.069745:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204895.069746:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204895.069747:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204895.069748:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204895.069750:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204895.069752:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204895.069757:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069761:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204895.069762:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069764:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.069765:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204895.069767:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069771:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204895.069773:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069777:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204895.069778:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069779:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.069780:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069781:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204895.069783:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204895.069784:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204895.069787:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204895.069788:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204895.069790:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204895.069795:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204895.069797:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204895.069799:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069804:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204895.069805:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.069807:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204895.069808:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204895.069809:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204895.069811:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204895.069814:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204895.069815:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204895.069815:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.069817:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204895.069819:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788204895.069820:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.069822:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204895.069823:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204895.069826:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204895.069827:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543485568, portal 4 00000100:00000001:3.0:1788204895.069829:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204895.069830:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922048 : -109424377629568 : ffff9c7aa6d77480) 00000100:00000040:3.0:1788204895.069834:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069838:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204895.069840:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204895.069841:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204895.069842:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543485568, offset 0 00000400:00000200:3.0:1788204895.069845:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204895.069847:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204895.069849:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204895.069850:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204895.069852:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204895.069854:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204895.069855:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204895.069857:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471142:471142:256:4294967295] 192.168.201.118@tcp LPNI seq info [471001:471001:8:4294967295] 00000400:00000200:3.0:1788204895.069860:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204895.069864:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204895.069868:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204895.069870:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204895.069872:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204895.069874:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204895.069876:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204895.069878:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788204895.069881:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204895.069882:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204895.069884:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204895.069886:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204895.069888:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204895.069889:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204895.069891:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204895.069893:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069898:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.069899:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069905:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069907:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204895.069909:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204895.069912:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204895.069916:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204895.069918:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321253917 00000100:00000001:3.0:1788204895.069920:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321255173 00000100:00000001:3.0:1788204895.069923:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321253917 00000100:00000001:3.0:1788204895.069925:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321255173 00000100:00000001:3.0:1788204895.069927:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204895.069929:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.069931:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204895.069936:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204895/real 1788204895] req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204895.069946:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204895.069947:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204895.069949:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204895.069952:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204895.069954:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204895.069957:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204895.069959:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204895.069965:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069969:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204895.069970:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069972:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.069973:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204895.069975:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069979:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204895.069982:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.069987:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204895.069988:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069989:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.069990:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.069991:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204895.069993:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204895.069994:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204895.069996:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204895.069997:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204895.070000:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204895.070005:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204895.070008:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204895.070010:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070014:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204895.070015:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.070016:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204895.070017:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204895.070018:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204895.070020:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204895.070022:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204895.070023:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204895.070024:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204895.070025:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204895.070027:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204895.070029:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204895.070031:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204895.070033:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204895.070035:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204895.070037:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543485696, portal 4 00000100:00000001:3.0:1788204895.070038:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204895.070039:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:3.0:1788204895.070044:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070048:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204895.070050:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204895.070051:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204895.070052:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543485696, offset 0 00000400:00000200:3.0:1788204895.070056:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204895.070058:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204895.070061:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204895.070063:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204895.070065:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204895.070067:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204895.070068:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204895.070070:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471143:471143:256:4294967295] 192.168.201.118@tcp LPNI seq info [471002:471002:8:4294967295] 00000400:00000200:3.0:1788204895.070073:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204895.070077:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204895.070080:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204895.070082:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204895.070084:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204895.070086:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788204895.070104:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204895.070107:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788204895.070110:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204895.070111:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204895.070113:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204895.070115:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204895.070117:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204895.070118:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204895.070120:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204895.070122:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070127:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.070129:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204920 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070134:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.070135:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204895.070137:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204895.070139:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204895.070140:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204895.070142:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321253941 00000100:00000001:3.0:1788204895.070144:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321255173 00000100:00000001:3.0:1788204895.070146:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321253941 00000100:00000001:3.0:1788204895.070147:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321255173 00000100:00000001:3.0:1788204895.070149:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204895.070150:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.070152:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204895.070155:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204895/real 1788204895] req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204895.070163:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204895.070165:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204895.070166:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204895.070168:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204895.070170:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204895.070173:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204895.070176:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204895.070186:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070194:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204895.070195:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070197:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.070199:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204895.070203:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070209:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204895.070212:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070217:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204895.070219:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070220:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204895.070221:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070222:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204895.070224:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204895.070225:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204895.070227:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204895.070229:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204895.070231:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204895.070236:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204895.070239:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070240:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.070242:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76680 x1875055543485312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070246:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204895.070247:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204895.070248:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788204895.070250:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204895.070251:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204895.070252:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204895.070254:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204895.070255:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d000. 02000000:00000001:3.0:1788204895.070257:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204895.070258:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204895.070259:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76680. 00000100:00000001:3.0:1788204895.070261:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204895.070262:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070264:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.070266:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75880 x1875055543485440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070275:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204895.070275:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204895.070277:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788204895.070278:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204895.070279:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204895.070280:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204895.070281:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204895.070282:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:3.0:1788204895.070283:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204895.070283:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204895.070284:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75880. 00000100:00000001:3.0:1788204895.070285:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204895.070286:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070287:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.070289:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77480 x1875055543485568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070293:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204895.070293:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204895.070294:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788204895.070295:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204895.070296:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204895.070297:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204895.070298:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204895.070299:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:3.0:1788204895.070300:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204895.070301:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204895.070302:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77480. 00000100:00000001:3.0:1788204895.070304:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204895.070305:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070305:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204895.070307:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543485696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204920 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204895.070311:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204895.070311:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204895.070313:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204895.070314:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204895.070315:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204895.070316:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204895.070317:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204895.070318:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:3.0:1788204895.070320:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204895.070320:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204895.070321:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:3.0:1788204895.070322:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204895.070323:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070324:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204895.070326:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204895.070327:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.070328:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204895.070330:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.900173:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204895.900178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204895.900179:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204895.900181:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204895.900184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204895.900187:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204895.900191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204895.900191:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204895.900195:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204895.900196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.900198:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204895.900202:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204895.900203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204895.900207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204895.900208:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204895.900216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204895.900219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204895.900278:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204895.900282:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204895.900286:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204896.412148:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321255173, at 4321255232 00000100:00000040:2.0:1788204896.412149:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321255173, at 4321255232 00000100:00000010:3.0:1788204896.412163:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc360. 00000100:00000010:2.0:1788204896.412166:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc840. 00000100:00000001:3.0:1788204896.924142:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204896.924148:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204896.924151:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204896.924154:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204896.924158:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204896.924159:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204896.924166:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204896.924167:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204896.924170:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204896.924171:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204896.924173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204896.924177:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204896.924177:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204896.924182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204896.924182:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204896.924185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204896.924187:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204896.924228:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204896.924232:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204896.924234:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204897.948194:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204897.948196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204897.948200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204897.948201:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204897.948204:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204897.948207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204897.948209:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204897.948211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204897.948214:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204897.948215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204897.948219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204897.948220:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204897.948223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204897.948227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204897.948227:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204897.948232:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204897.948236:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204897.948254:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204897.948259:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204897.948263:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204898.972186:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204898.972190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204898.972197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788204898.972197:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204898.972202:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204898.972203:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204898.972206:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204898.972211:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204898.972212:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204898.972215:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204898.972218:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.972218:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204898.972224:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.972225:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204898.972228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204898.972232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.972232:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204898.972238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204898.972238:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204898.972241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204898.972243:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204898.972245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204898.972247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204898.972300:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204898.972305:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204898.972308:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204899.996174:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204899.996184:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204899.996186:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204899.996193:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204899.996194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204899.996200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204899.996200:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204899.996207:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204899.996210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204899.996213:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204899.996214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204899.996218:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204899.996222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204899.996225:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204899.996227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204899.996235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204899.996239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204899.996317:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204899.996323:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204899.996327:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204900.188240:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204900.188251:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204900.188254:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204900.188258:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204900.188262:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204900.188270:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26586 00000400:00000200:2.0:1788204900.188280:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204900.188288:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204900.188291:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204900.188294:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188297:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204900.188299:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188301:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204900.188305:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204900.188307:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204900.188310:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204900.188313:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204900.188315:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204900.188317:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204900.188319:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188320:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204900.188323:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188324:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204900.188328:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204900.188332:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188335:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204900.188338:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204900.188343:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204900.188346:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188349:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204900.188352:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204900.188365:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204900.188370:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204900.188376:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204900.188378:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188382:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204900.188385:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204900.188392:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204900.188400:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204900.188407:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204900.188415:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204900.188424:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204900.188428:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204900.188432:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204900.188453:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204900.188455:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204900.188458:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204900.188459:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204900.188460:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204900.188462:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204900.188466:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26586 00000400:00000200:2.0:1788204900.188469:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204900.188471:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204900.188473:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204900.188475:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188477:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204900.188479:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188480:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204900.188483:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204900.188484:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204900.188485:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204900.188487:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204900.188488:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204900.188490:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204900.188492:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188494:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204900.188495:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188497:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204900.188499:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204900.188501:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188502:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204900.188519:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204900.188525:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204900.188527:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188528:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204900.188532:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204900.188535:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.188536:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204900.188537:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204900.188542:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000100:00000001:3.0:1788204900.188544:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1788204900.188546:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000001:3.0:1788204900.188548:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788204900.188550:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204900.188553:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188556:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204900.188558:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1788204900.188558:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204900.188562:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204900.188568:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:3.0:1788204900.188574:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204900.188575:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.188578:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204900.188583:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788204900.188583:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204900.188588:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204900.188592:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188598:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788204900.188599:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543485824:192.168.201.118@tcp:250:kworker.0 00000100:00000001:2.0:1788204900.188602:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.188609:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1788204900.188613:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204900.188615:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204900.188618:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.188623:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788204900.188631:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204900.188631:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204900.188635:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204900.188638:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788204900.188644:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:2.0:1788204900.188648:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:3.0:1788204900.188650:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000100:00000001:2.0:1788204900.188660:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:3.0:1788204900.188660:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00080000:2.0:1788204900.188666:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204900.188669:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788204900.188671:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543485824, portal 25 00000100:00080000:2.0:1788204900.188674:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26586 00000100:00000001:3.0:1788204900.188676:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204900.188679:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000400:00000200:2.0:1788204900.188682:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204900.188689:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000040:3.0:1788204900.188690:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204900.188691:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204900.188698:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188704:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204900.188706:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788204900.188707:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188712:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788204900.188712:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000020:00000001:2.0:1788204900.188717:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000010:3.0:1788204900.188718:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000020:00000040:2.0:1788204900.188719:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204900.188722:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:3.0:1788204900.188722:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543485824, offset 0 00000020:00000040:2.0:1788204900.188726:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204900.188728:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788204900.188733:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000020:00000001:2.0:1788204900.188734:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204900.188739:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188740:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204900.188742:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188744:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204900.188747:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:2.0:1788204900.188751:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788204900.188757:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00080000:2.0:1788204900.188758:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188764:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204900.188765:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1788204900.188770:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788204900.188773:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000001:2.0:1788204900.188776:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204900.188780:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188783:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204900.188784:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000008:00100000:2.0:1788204900.188787:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788204900.188789:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000008:00000001:2.0:1788204900.188793:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188798:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204900.188800:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471144:471144:256:4294967295] 192.168.201.118@tcp LPNI seq info [471003:471003:8:4294967295] 00000100:00000010:2.0:1788204900.188804:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000400:00000200:3.0:1788204900.188810:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:2.0:1788204900.188811:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204900.188817:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204900.188818:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188820:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204900.188822:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204900.188823:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204900.188831:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000010:2.0:1788204900.188836:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204900.188843:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204900.188848:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788204900.188852:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188855:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204900.188859:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204900.188861:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1788204900.188863:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000100:00000001:2.0:1788204900.188874:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204900.188881:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.0:1788204900.188881:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204900.188890:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:3.0:1788204900.188891:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:2.0:1788204900.188892:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204900.188896:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204900.188903:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.1:1788204900.188903:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204900.188910:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26586 00000800:00000200:3.1:1788204900.188911:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000400:00000200:2.0:1788204900.188920:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000100:3.0:1788204900.188923:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000400:00000200:2.0:1788204900.188927:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000010:3.0:1788204900.188931:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000100:00080000:2.0:1788204900.188932:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204900.188937:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188940:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204900.188941:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188942:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204900.188944:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000200:3.0:1788204900.188944:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:2.0:1788204900.188947:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204900.188950:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204900.188952:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000400:00000200:3.0:1788204900.188956:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1788204900.188957:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204900.188961:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204900.188966:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.188967:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204900.188968:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000100:00000040:2.0:1788204900.188970:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:3.0:1788204900.188972:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:2.0:1788204900.188973:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204900.188978:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204900.188980:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204900.188981:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1788204900.188983:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204900.188985:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204900.188989:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1788204900.188989:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204900.188995:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.188999:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204900.189003:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:3.0:1788204900.189006:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788204900.189007:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.189011:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204900.189014:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204900.189016:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204900.189021:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204900.189026:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204900.189028:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189030:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.189031:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204900.189033:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1788204900.189034:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204900.189037:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204900.189041:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:3.0:1788204900.189042:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204900.189044:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204900.189049:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:2.0:1788204900.189052:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204900.189054:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321260293 00000100:00000001:2.0:1788204900.189064:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:3.0:1788204900.189064:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc360. 00000100:00000001:2.0:1788204900.189068:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204900.189072:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189075:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1788204900.189080:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204900.189084:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204900.189086:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321260293 00000100:00080000:2.0:1788204900.189087:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000010:3.0:1788204900.189120:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcc60. 00000100:00000001:2.0:1788204900.189122:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204900.189126:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204900.189127:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1788204900.189133:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204900.189135:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788204900.189140:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26586 00000100:00000001:3.0:1788204900.189142:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189148:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1788204900.189149:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204900.189158:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00100000:3.0:1788204900.189158:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204900/real 1788204900] req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204900.189163:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204900.189171:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.189176:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204900.189179:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000001:3.0:1788204900.189181:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788204900.189183:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000001:3.0:1788204900.189184:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1788204900.189186:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000010:3.0:1788204900.189188:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000020:00000040:2.0:1788204900.189189:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204900.189193:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1788204900.189195:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000020:00000040:2.0:1788204900.189196:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000400:00000010:3.0:1788204900.189199:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000020:00000001:2.0:1788204900.189200:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204900.189204:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788204900.189204:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1788204900.189212:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000200:3.0:1788204900.189212:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204900.189217:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204900.189221:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204900.189226:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204900.189231:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000200:3.0:1788204900.189234:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204900.189236:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204900.189241:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204900.189245:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204900.189250:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788204900.189251:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204900.189252:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204900.189258:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189258:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.189261:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204900.189264:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:3.0:1788204900.189267:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788204900.189269:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204900.189276:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189278:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204900.189281:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204900.189285:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204900.189289:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000100:00000001:3.0:1788204900.189293:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1788204900.189295:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788204900.189297:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204900.189299:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204900.189300:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788204900.189303:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189304:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204900.189307:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204900.189309:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788204900.189310:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204900.189312:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788204900.189313:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204900.189317:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000010:2.0:1788204900.189320:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00080000:3.0:1788204900.189323:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:2.0:1788204900.189327:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189327:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204900.189335:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204900.189338:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204900.189356:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543485824:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788204900.189371:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204900.189374:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204900.189374:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788204900.189378:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204900.189380:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543485824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204900.189385:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204900.189404:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189404:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204900.189410:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1788204900.189411:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26596) 02000000:00000010:3.0:1788204900.189416:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204900.189421:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204900.189423:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204900.189425:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204900.189428:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204900.189432:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d800. 02000000:00000001:3.0:1788204900.189438:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204900.189439:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204900.189441:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:3.0:1788204900.189445:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204900.189447:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.189449:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.189458:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204900.189459:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189461:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204900.189462:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204900.189463:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204900.189467:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189472:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204900.189474:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.189475:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204900.189477:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204900.189478:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204900.189481:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204900.189483:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204900.189485:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204900.189486:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.189488:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204900.189490:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204900.189492:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204900.189496:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204900.189498:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204900.189500:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204900.189503:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543485952, portal 10 00000100:00000001:3.0:1788204900.189523:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204900.189524:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000040:3.0:1788204900.189531:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189536:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204900.189537:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204900.189539:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204900.189542:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543485952, offset 0 00000400:00000200:3.0:1788204900.189545:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204900.189548:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204900.189552:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204900.189556:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204900.189559:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204900.189561:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204900.189564:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204900.189567:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471145:471145:256:4294967295] 192.168.201.118@tcp LPNI seq info [471004:471004:8:4294967295] 00000400:00000200:3.0:1788204900.189572:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204900.189579:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204900.189584:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204900.189588:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204900.189592:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204900.189595:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204900.189600:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204900.189603:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788204900.189608:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204900.189610:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204900.189613:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204900.189618:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204900.189621:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204900.189623:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204900.189626:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204900.189629:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189636:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204900.189638:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189643:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189644:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204900.189647:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204900.189650:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204900.189653:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204900.189654:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321259027 00000100:00000001:3.0:1788204900.189657:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321260293 00000100:00000001:3.0:1788204900.189658:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321259027 00000100:00000001:3.0:1788204900.189660:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321260293 00000100:00000001:3.0:1788204900.189662:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204900.189663:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189665:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204900.189667:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204900/real 1788204900] req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204900.189672:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204900.189673:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204900.189674:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204900.189676:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204900.189677:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204900.189679:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204900.189681:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204900.189687:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189691:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204900.189692:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.189693:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.189694:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204900.189696:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189700:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204900.189703:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189707:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204900.189708:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.189709:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.189710:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.189711:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204900.189713:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204900.189715:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204900.189717:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204900.189718:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204900.189722:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204900.189731:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543485952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204900.189736:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204900.189739:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189752:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204900.189754:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.189756:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204900.189758:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204900.189761:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204900.189764:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204900.189768:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204900.189770:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204900.189772:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.189774:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204900.189778:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788204900.189780:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204900.189784:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204900.189787:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204900.189791:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204900.189794:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543486080, portal 10 00000100:00000001:3.0:1788204900.189796:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204900.189798:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 00000100:00000040:3.0:1788204900.189804:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189811:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204900.189814:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204900.189817:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204900.189819:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543486080, offset 0 00000400:00000200:3.0:1788204900.189822:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204900.189825:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204900.189828:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204900.189830:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204900.189832:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204900.189834:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204900.189836:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204900.189839:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471146:471146:256:4294967295] 192.168.201.118@tcp LPNI seq info [471005:471005:8:4294967295] 00000400:00000200:3.0:1788204900.189842:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204900.189847:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204900.189850:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204900.189852:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204900.189855:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204900.189857:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204900.189860:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204900.189863:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788204900.189865:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204900.189867:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204900.189869:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204900.189872:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204900.189874:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204900.189875:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204900.189877:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204900.189879:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189885:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204900.189887:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189892:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189893:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204900.189895:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204900.189897:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204900.189899:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204900.189901:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321259028 00000100:00000001:3.0:1788204900.189903:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321260293 00000100:00000001:3.0:1788204900.189904:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321259028 00000100:00000001:3.0:1788204900.189906:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321260293 00000100:00000001:3.0:1788204900.189907:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204900.189909:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.189911:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204900.189913:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204900/real 1788204900] req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204900.189919:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204900.189919:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204900.189920:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204900.189922:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204900.189923:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204900.189924:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204900.189926:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204900.189932:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189936:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204900.189937:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.189938:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.189939:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204900.189941:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189945:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204900.189947:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189951:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204900.189953:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.189954:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.189954:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.189956:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204900.189958:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204900.189959:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204900.189962:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204900.189963:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204900.189966:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204900.189972:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204900.189976:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204900.189979:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.189987:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204900.189988:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.189991:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204900.189993:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204900.189995:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204900.189998:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204900.190002:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204900.190004:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204900.190006:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.190008:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204900.190012:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788204900.190014:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204900.190018:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204900.190021:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204900.190025:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204900.190028:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543486208, portal 4 00000100:00000001:3.0:1788204900.190031:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204900.190032:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913984 : -109424377637632 : ffff9c7aa6d75500) 00000100:00000040:3.0:1788204900.190039:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190046:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204900.190049:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204900.190051:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204900.190054:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543486208, offset 0 00000400:00000200:3.0:1788204900.190058:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204900.190062:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204900.190066:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204900.190069:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204900.190071:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204900.190074:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204900.190076:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204900.190080:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471147:471147:256:4294967295] 192.168.201.118@tcp LPNI seq info [471006:471006:8:4294967295] 00000400:00000200:3.0:1788204900.190084:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204900.190111:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204900.190117:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204900.190122:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204900.190126:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204900.190129:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204900.190134:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204900.190137:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788204900.190142:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204900.190144:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204900.190148:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204900.190152:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204900.190155:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204900.190157:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204900.190160:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204900.190165:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190174:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204900.190177:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190185:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.190187:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204900.190191:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204900.190194:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204900.190198:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204900.190201:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321259038 00000100:00000001:3.0:1788204900.190204:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321260293 00000100:00000001:3.0:1788204900.190207:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321259038 00000100:00000001:3.0:1788204900.190209:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321260293 00000100:00000001:3.0:1788204900.190212:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204900.190214:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.190217:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204900.190221:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204900/real 1788204900] req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204900.190230:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204900.190232:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204900.190234:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204900.190237:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204900.190238:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204900.190241:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204900.190244:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204900.190254:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190262:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204900.190264:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190266:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.190268:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204900.190272:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190280:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204900.190284:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190293:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204900.190296:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190298:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.190299:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190301:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204900.190304:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204900.190307:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204900.190310:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204900.190313:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204900.190317:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204900.190326:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204900.190331:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204900.190335:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190343:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204900.190344:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.190347:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204900.190349:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204900.190352:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204900.190355:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204900.190359:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204900.190361:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204900.190363:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204900.190365:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204900.190369:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788204900.190371:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204900.190375:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204900.190378:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 00000400:00000010:3.0:1788204900.190382:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788204900.190385:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543486336, portal 4 00000100:00000001:3.0:1788204900.190387:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204900.190388:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331909504 : -109424377642112 : ffff9c7aa6d74380) 00000100:00000040:3.0:1788204900.190395:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190403:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204900.190405:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204900.190408:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000100:00000200:3.0:1788204900.190411:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543486336, offset 0 00000400:00000200:3.0:1788204900.190416:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204900.190420:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204900.190425:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204900.190428:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204900.190431:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204900.190433:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204900.190436:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204900.190440:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471148:471148:256:4294967295] 192.168.201.118@tcp LPNI seq info [471007:471007:8:4294967295] 00000400:00000200:3.0:1788204900.190445:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204900.190453:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204900.190458:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204900.190461:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788204900.190466:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204900.190468:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788204900.190474:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204900.190477:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788204900.190481:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204900.190483:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788204900.190487:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204900.190491:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204900.190494:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:3.0:1788204900.190497:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:3.0:1788204900.190500:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204900.190504:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190528:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204900.190532:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204925 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190539:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.190541:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204900.190545:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204900.190549:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204900.190553:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204900.190555:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321259061 00000100:00000001:3.0:1788204900.190558:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321260293 00000100:00000001:3.0:1788204900.190561:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321259061 00000100:00000001:3.0:1788204900.190564:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321260293 00000100:00000001:3.0:1788204900.190566:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204900.190569:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.190571:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204900.190575:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204900/real 1788204900] req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204900.190584:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204900.190585:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204900.190587:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77aa8. 00000400:00000200:3.0:1788204900.190590:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:3.0:1788204900.190592:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:3.0:1788204900.190594:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204900.190598:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204900.190609:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190623:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204900.190625:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190627:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.190629:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204900.190633:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190641:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204900.190646:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190654:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204900.190657:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190659:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204900.190661:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190663:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204900.190665:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204900.190668:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204900.190672:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204900.190675:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204900.190679:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204900.190689:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204900.190694:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190696:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204900.190700:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543485952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190708:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204900.190710:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204900.190713:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204900.190716:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204900.190717:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204900.190719:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204900.190722:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204900.190724:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:3.0:1788204900.190727:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204900.190729:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204900.190731:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:3.0:1788204900.190734:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204900.190736:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190737:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204900.190741:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543486080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190749:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204900.190751:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204900.190752:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788204900.190755:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204900.190756:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204900.190758:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204900.190761:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204900.190762:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:3.0:1788204900.190765:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204900.190766:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204900.190768:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:3.0:1788204900.190771:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204900.190772:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190774:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204900.190778:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75500 x1875055543486208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190786:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204900.190787:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204900.190789:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788204900.190791:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204900.190793:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204900.190794:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204900.190797:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204900.190798:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788204900.190801:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204900.190802:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204900.190803:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75500. 00000100:00000001:3.0:1788204900.190806:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204900.190807:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190809:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204900.190813:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74380 x1875055543486336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204925 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204900.190821:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204900.190822:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204900.190823:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788204900.190826:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204900.190827:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204900.190829:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204900.190831:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204900.190833:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:3.0:1788204900.190836:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204900.190837:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204900.190839:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74380. 00000100:00000001:3.0:1788204900.190841:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204900.190842:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190845:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204900.190849:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204900.190851:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204900.190853:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204900.190855:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204901.020213:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204901.020218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204901.020222:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204901.020223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204901.020226:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204901.020229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204901.020231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204901.020233:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204901.020235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204901.020236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204901.020241:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204901.020242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204901.020243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204901.020248:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204901.020254:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204901.020258:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204901.020261:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204901.020270:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204901.020273:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204901.020275:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204901.532226:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321260293, at 4321260352 00000100:00000040:3.0:1788204901.532226:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321260293, at 4321260352 00000100:00000010:2.0:1788204901.532245:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc360. 00000100:00000010:3.0:1788204901.532248:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcc60. 00010000:00000001:0.0:1788204902.044218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204902.044223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204902.044231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204902.044234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204902.044241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204902.044245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204902.044249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204902.044254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204902.044274:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204902.044284:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204902.044287:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204902.044293:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204902.044299:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204902.044304:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204902.044308:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204902.044311:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204902.044314:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204902.044321:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204902.044325:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204902.044330:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204903.068223:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204903.068233:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204903.068235:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204903.068241:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204903.068247:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204903.068249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204903.068251:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204903.068256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204903.068260:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204903.068265:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204903.068266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204903.068270:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204903.068271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204903.068286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204903.068291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204903.068303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204903.068313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204903.068385:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204903.068389:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204903.068391:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204904.092235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204904.092241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204904.092255:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204904.092259:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204904.092264:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092269:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204904.092272:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204904.092281:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204904.092282:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204904.092283:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092285:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204904.092286:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092287:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092289:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204904.092294:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204904.092294:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204904.092296:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092297:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204904.092299:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092300:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204904.092302:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204904.092324:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204904.092327:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204904.092329:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204904.092403:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204904.092408:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204904.092410:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204904.092412:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204904.092415:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204904.092417:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204904.092420:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204904.092422:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204904.092423:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.116147:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204905.116155:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204905.116157:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204905.116162:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204905.116166:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204905.116172:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204905.116176:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204905.116180:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204905.116186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204905.116186:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204905.116191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204905.116198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204905.116201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204905.116206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204905.116208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204905.116211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204905.116212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204905.116267:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204905.116281:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204905.116289:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204905.308201:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204905.308210:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204905.308212:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204905.308215:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204905.308219:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204905.308226:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26591 00000400:00000200:2.0:1788204905.308235:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204905.308242:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204905.308245:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204905.308248:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308251:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204905.308253:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308254:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204905.308258:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204905.308260:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204905.308262:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204905.308266:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204905.308267:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204905.308269:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204905.308272:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308273:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204905.308274:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308276:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204905.308278:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204905.308280:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308282:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204905.308284:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204905.308287:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204905.308289:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308290:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204905.308292:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204905.308300:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204905.308303:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204905.308306:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204905.308307:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308310:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204905.308312:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204905.308316:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204905.308322:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204905.308328:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204905.308337:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543486464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204905.308346:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204905.308350:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204905.308353:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543486464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204905.308371:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204905.308373:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204905.308376:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204905.308377:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204905.308378:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204905.308380:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204905.308383:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26591 00000400:00000200:2.0:1788204905.308386:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204905.308388:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204905.308391:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204905.308391:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788204905.308396:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204905.308398:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204905.308399:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204905.308402:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204905.308408:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308412:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788204905.308412:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76a00 x1875055543486464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204905.308428:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204905.308430:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204905.308434:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204905.308437:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204905.308441:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204905.308446:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788204905.308446:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204905.308451:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204905.308452:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308457:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204905.308459:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308461:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:3.0:1788204905.308461:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:2.0:1788204905.308465:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:3.0:1788204905.308466:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788204905.308469:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788204905.308473:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308474:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204905.308479:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788204905.308483:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543486464:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788204905.308485:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204905.308490:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.308490:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788204905.308494:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204905.308496:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1788204905.308498:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204905.308499:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788204905.308502:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.308503:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788204905.308506:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204905.308511:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413000. 00000100:00000010:2.0:1788204905.308512:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204905.308517:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204905.308517:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.308527:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788204905.308528:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204905.308531:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308533:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204905.308535:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1788204905.308536:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77aa8. 02000000:00000001:2.0:1788204905.308537:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204905.308545:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000400:00000010:3.0:1788204905.308549:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000001:2.0:1788204905.308550:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204905.308557:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543486592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788204905.308561:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543486464, portal 25 00000100:00000001:2.0:1788204905.308565:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204905.308568:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204905.308569:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.308573:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331919360 : -109424377632256 : ffff9c7aa6d76a00) 00000100:00000040:2.0:1788204905.308574:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543486592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204905.308585:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204905.308587:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76a00 x1875055543486464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204905.308589:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204905.308596:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204905.308600:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204905.308603:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204905.308608:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204905.308608:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788204905.308614:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26591 00000100:00000040:3.0:1788204905.308615:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204905.308620:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 00000400:00000200:2.0:1788204905.308621:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:3.0:1788204905.308627:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543486464, offset 0 00000400:00000200:2.0:1788204905.308628:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204905.308632:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204905.308639:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204905.308639:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308643:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204905.308646:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308647:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204905.308649:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204905.308651:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:3.0:1788204905.308652:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:2.0:1788204905.308654:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204905.308657:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204905.308659:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788204905.308662:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:2.0:1788204905.308663:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204905.308670:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308671:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204905.308672:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788204905.308676:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308680:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204905.308681:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000020:00000040:2.0:1788204905.308685:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788204905.308688:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1788204905.308692:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:3.0:1788204905.308695:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308696:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204905.308699:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788204905.308703:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471149:471149:256:4294967295] 192.168.201.118@tcp LPNI seq info [471008:471008:8:4294967295] 02000000:00000001:2.0:1788204905.308704:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204905.308710:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308713:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:3.0:1788204905.308714:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00100000:2.0:1788204905.308717:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204905.308720:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308721:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204905.308724:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204905.308725:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204905.308727:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204905.308728:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308729:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204905.308730:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204905.308731:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204905.308734:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788204905.308739:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.308742:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00100000:2.0:1788204905.308747:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543486720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788204905.308754:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308762:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:3.0:1788204905.308763:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000100:00000001:2.0:1788204905.308766:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204905.308771:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788204905.308775:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543486720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788204905.308780:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204905.308791:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204905.308795:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204905.308796:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204905.308801:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788204905.308804:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204905.308806:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204905.308808:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321265413 00000100:00080000:2.0:1788204905.308810:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:3.0:1788204905.308815:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcc60. 00000100:00000001:2.0:1788204905.308816:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204905.308822:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26591 00000100:00000001:3.0:1788204905.308824:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000400:00000200:2.0:1788204905.308828:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204905.308830:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321265413 00000400:00000200:2.0:1788204905.308834:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204905.308837:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000010:3.0:1788204905.308837:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc120. 00000100:00000040:2.0:1788204905.308844:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204905.308844:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1788204905.308850:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204905.308852:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788204905.308855:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204905.308857:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308858:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204905.308861:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:3.0:1788204905.308862:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204905.308863:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204905.308867:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788204905.308868:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1788204905.308870:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204905.308872:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204905.308874:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:3.0:1788204905.308876:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1788204905.308877:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308881:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204905.308882:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000040:2.0:1788204905.308884:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204905.308888:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204905.308889:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204905.308893:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204905.308896:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204905.308899:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204905.308901:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204905.308903:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788204905.308906:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204905.308910:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.308912:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204905.308914:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204905.308919:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000040:3.0:1788204905.308921:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74a80 x1875055543486592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788204905.308924:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.308930:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204905.308939:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000100:00000001:3.0:1788204905.308940:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:2.0:1788204905.308944:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788204905.308944:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.308951:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204905.308952:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204905.308956:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.308957:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204905.308959:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204905.308962:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204905.308963:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204905.308967:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204905.308971:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486592:192.168.201.118@tcp:38:kworker.0 02000000:00000010:2.0:1788204905.308975:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:3.0:1788204905.308978:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788204905.308982:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.308983:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204905.308988:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.308992:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788204905.308993:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543486848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:3.0:1788204905.308998:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788204905.309004:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.309007:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204905.309012:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204905.309013:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204905.309020:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77f20. 00000100:00000040:2.0:1788204905.309021:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543486848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788204905.309028:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987346e8. 00000100:00000200:3.0:1788204905.309034:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543486592, portal 10 00000100:00000001:2.0:1788204905.309036:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.309038:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788204905.309041:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204905.309042:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331911296 : -109424377640320 : ffff9c7aa6d74a80) 00000100:00080000:2.0:1788204905.309050:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204905.309052:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204905.309056:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1788204905.309057:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74a80 x1875055543486592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204905.309061:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204905.309068:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788204905.309073:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26591 00000100:00000040:3.0:1788204905.309077:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000200:2.0:1788204905.309082:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204905.309082:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000400:00000200:2.0:1788204905.309110:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204905.309114:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204905.309118:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.309121:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204905.309124:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.309126:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204905.309129:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204905.309131:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204905.309133:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204905.309135:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000200:3.0:1788204905.309135:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543486592, offset 0 00000020:00000001:2.0:1788204905.309140:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204905.309144:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000200:3.0:1788204905.309150:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000040:2.0:1788204905.309151:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.309156:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204905.309159:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788204905.309160:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204905.309166:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204905.309169:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000200:3.0:1788204905.309170:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00080000:2.0:1788204905.309177:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:3.0:1788204905.309179:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788204905.309183:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.309187:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000001:2.0:1788204905.309188:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788204905.309194:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1788204905.309195:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204905.309201:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.309204:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204905.309206:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000400:00000200:3.0:1788204905.309208:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000008:00000001:2.0:1788204905.309211:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.309218:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.309218:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471150:471150:255:4294967295] 192.168.201.118@tcp LPNI seq info [471009:471009:7:4294967295] 00000100:00000010:2.0:1788204905.309226:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000400:00000200:3.0:1788204905.309229:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:2.0:1788204905.309232:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204905.309238:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204905.309241:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.309243:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:2.0:1788204905.309245:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204905.309249:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204905.309253:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204905.309255:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000010:2.0:1788204905.309259:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000800:00000010:3.0:1788204905.309262:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd100. 00000100:00000001:2.0:1788204905.309265:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204905.309272:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00100000:2.0:1788204905.309277:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543486976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204905.309283:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788204905.309291:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204905.309297:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204905.309301:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204905.309307:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543486976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204905.309315:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000100:00000001:2.0:1788204905.309316:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.309325:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788204905.309326:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26601) 00000100:00000001:3.0:1788204905.309330:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321264147 00000100:00000001:3.0:1788204905.309335:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321265413 00000100:00000001:3.0:1788204905.309338:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321264147 00000100:00000001:3.0:1788204905.309341:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321265413 00000100:00000001:3.0:1788204905.309344:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204905.309347:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.309350:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204905.309356:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543486720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204905.309365:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204905.309367:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.309369:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204905.309371:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204905.309374:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204905.309377:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486720:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204905.309381:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204905.309383:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204905.309385:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.309387:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204905.309391:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788204905.309394:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.309397:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204905.309406:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77948. 00000400:00000010:3.0:1788204905.309411:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987342a8. 00000100:00000200:3.0:1788204905.309415:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543486720, portal 10 00000100:00000001:3.0:1788204905.309429:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204905.309430:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:3.0:1788204905.309438:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543486720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204905.309445:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204905.309448:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204905.309451:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734e58. 00000100:00000200:3.0:1788204905.309454:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543486720, offset 0 00000400:00000200:3.0:1788204905.309460:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204905.309464:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204905.309468:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204905.309471:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204905.309474:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204905.309477:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204905.309480:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204905.309484:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471151:471151:254:4294967295] 192.168.201.118@tcp LPNI seq info [471010:471010:6:4294967295] 00000400:00000200:3.0:1788204905.309489:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204905.309496:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204905.309502:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204905.309506:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd400. 00000800:00000200:3.0:1788204905.309510:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204905.309514:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788204905.309518:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204905.309521:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204905.309524:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204905.309526:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321264148 00000100:00000001:3.0:1788204905.309528:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321265413 00000100:00000001:3.0:1788204905.309531:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321264148 00000100:00000001:3.0:1788204905.309533:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321265413 00000100:00000001:3.0:1788204905.309535:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204905.309538:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.309541:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.309543:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204905.309547:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204905.309549:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788204905.309550:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204905.309552:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204905.309553:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204905.309558:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74380 x1875055543486848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204905.309566:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204905.309567:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.309569:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204905.309572:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204905.309574:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204905.309577:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486848:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204905.309580:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204905.309582:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204905.309584:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.309586:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204905.309589:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788204905.309592:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.309596:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204905.309599:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77c60. 00000400:00000010:3.0:1788204905.309603:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734110. 00000100:00000200:3.0:1788204905.309606:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543486848, portal 4 00000100:00000001:3.0:1788204905.309608:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204905.309610:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331909504 : -109424377642112 : ffff9c7aa6d74380) 00000100:00000040:3.0:1788204905.309618:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74380 x1875055543486848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204905.309625:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204905.309627:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204905.309630:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788204905.309633:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543486848, offset 0 00000400:00000200:3.0:1788204905.309638:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204905.309642:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204905.309646:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204905.309649:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204905.309651:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204905.309654:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204905.309657:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204905.309660:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471152:471152:253:4294967295] 192.168.201.118@tcp LPNI seq info [471011:471011:5:4294967295] 00000400:00000200:3.0:1788204905.309665:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204905.309672:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204905.309677:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204905.309681:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788204905.309686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204905.309690:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788204905.309693:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204905.309697:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204905.309700:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204905.309703:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321264157 00000100:00000001:3.0:1788204905.309705:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321265413 00000100:00000001:3.0:1788204905.309708:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321264157 00000100:00000001:3.0:1788204905.309710:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321265413 00000100:00000001:3.0:1788204905.309712:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204905.309714:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.309717:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204905.309721:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75500 x1875055543486976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204905.309730:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204905.309731:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.309734:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204905.309736:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204905.309738:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204905.309742:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543486976:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204905.309746:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204905.309748:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204905.309750:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204905.309752:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204905.309755:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788204905.309757:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204905.309761:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204905.309763:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77c08. 00000400:00000010:3.0:1788204905.309778:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987345d8. 00000100:00000200:3.0:1788204905.309782:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543486976, portal 4 00000100:00000001:3.0:1788204905.309785:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204905.309787:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913984 : -109424377637632 : ffff9c7aa6d75500) 00000100:00000040:3.0:1788204905.309799:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75500 x1875055543486976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204905.309806:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204905.309809:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204905.309812:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788204905.309815:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543486976, offset 0 00000400:00000200:3.0:1788204905.309897:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204905.309902:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204905.309906:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204905.309909:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204905.309912:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204905.309914:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204905.309916:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204905.309938:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471153:471153:252:4294967295] 192.168.201.118@tcp LPNI seq info [471012:471012:4:4294967295] 00000400:00000200:3.0:1788204905.309943:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204905.309950:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204905.309955:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204905.309960:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd900. 00000800:00000200:3.0:1788204905.309966:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204905.309970:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788204905.309973:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204905.309977:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204905.309980:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204905.309982:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321264180 00000100:00000001:3.0:1788204905.309985:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321265413 00000100:00000001:3.0:1788204905.309988:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321264180 00000100:00000001:3.0:1788204905.309991:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321265413 00000100:00000001:3.0:1788204905.309993:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204905.309995:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.309998:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.310000:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204905.310005:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204905.310007:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788204905.310009:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204905.310010:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204905.310013:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204905.310015:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204906.140198:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204906.140206:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204906.140208:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204906.140212:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204906.140215:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204906.140218:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204906.140220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204906.140224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204906.140228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204906.140229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204906.140231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204906.140232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204906.140234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204906.140235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204906.140236:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204906.140240:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204906.140243:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204906.140295:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204906.140298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204906.140302:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204906.652285:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321265413, at 4321265472 00000100:00000040:2.0:1788204906.652288:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321265413, at 4321265472 00000100:00000010:2.0:1788204906.652312:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcc60. 00000100:00000010:3.0:1788204906.652318:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc120. 00000100:00000001:3.0:1788204907.164157:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204907.164166:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204907.164169:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204907.164176:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204907.164181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204907.164183:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204907.164188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204907.164190:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204907.164197:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204907.164198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204907.164202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204907.164202:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204907.164207:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204907.164211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204907.164215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204907.164224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204907.164228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204907.164253:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204907.164256:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204907.164259:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204908.188200:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204908.188210:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204908.188213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204908.188215:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204908.188218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204908.188223:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204908.188226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204908.188229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.188230:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204908.188236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204908.188237:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204908.188241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.188243:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204908.188248:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204908.188249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204908.188254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204908.188266:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204908.188326:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204908.188330:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204908.188333:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788204908.380720:0:256120:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:0.0:1788204908.380802:0:256120:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788204908.380812:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204908.380815:0:256120:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:0.0:1788204908.380820:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204908.380835:0:256120:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:0.0:1788204908.380842:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204908.380846:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:0.0:1788204908.380853:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204908.380858:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204908.380877:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000400:00000010:0.0:1788204908.380881:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:0.0:1788204908.381614:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204908.381637:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d76a00 x1875055543486464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.381691:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204908.381696:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543486464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.381712:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.381715:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204908.381750:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204908.381754:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd100. 00000400:00000200:0.0:1788204908.381758:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204908.381770:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204908.381776:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:0.0:1788204908.381779:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:0.0:1788204908.381784:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204908.381797:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74a80 x1875055543486592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.381833:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204908.381838:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74a80 x1875055543486592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.381845:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.381847:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204908.381861:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204908.381864:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd400. 00000400:00000200:0.0:1788204908.381868:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204908.381872:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204908.381876:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734e58 00000400:00000010:0.0:1788204908.381878:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734e58. 00000100:00000001:0.0:1788204908.381880:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204908.381892:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d77b80 x1875055543486720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.381901:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204908.381904:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543486720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.381918:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.381920:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204908.381928:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204908.381931:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:0.0:1788204908.381935:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204908.381940:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204908.381950:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:0.0:1788204908.381952:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:0.0:1788204908.381956:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204908.381962:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74380 x1875055543486848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.381977:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204908.381981:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74380 x1875055543486848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.382002:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.382004:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788204908.382016:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788204908.382019:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd900. 00000400:00000200:0.0:1788204908.382022:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788204908.382026:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788204908.382030:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:0.0:1788204908.382032:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:0.0:1788204908.382035:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788204908.382047:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75500 x1875055543486976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.382056:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788204908.382060:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75500 x1875055543486976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204930 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204908.382073:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204908.382075:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204909.212202:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204909.212212:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204909.212216:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204909.212225:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204909.212230:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204909.212235:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204909.212242:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204909.212246:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204909.212250:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204909.213488:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204909.213495:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204909.213497:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204909.213501:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204909.213506:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204909.213508:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204909.213511:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204909.213513:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204909.213520:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204909.213521:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204909.213525:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204909.213527:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204909.213531:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204909.213532:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204909.213618:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204909.213621:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204909.213624:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204910.236187:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204910.236190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204910.236195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.236196:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204910.236201:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204910.236203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204910.236207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204910.236209:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204910.236211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204910.236215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.236216:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204910.236219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204910.236222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.236222:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204910.236230:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204910.236235:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204910.236238:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204910.236255:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204910.236258:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204910.236261:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204910.428192:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204910.428205:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204910.428208:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204910.428212:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204910.428216:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204910.428223:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26596 00000400:00000200:3.0:1788204910.428234:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204910.428242:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204910.428246:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204910.428249:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428253:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.428256:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428259:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204910.428263:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204910.428265:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788204910.428269:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204910.428272:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788204910.428275:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788204910.428277:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788204910.428281:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428283:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.428285:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428288:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204910.428290:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788204910.428294:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428297:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204910.428300:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204910.428304:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204910.428307:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.428309:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204910.428312:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204910.428319:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788204910.428323:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788204910.428327:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204910.428347:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.428351:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204910.428354:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204910.428359:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204910.428367:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412c00. 00000100:00000001:3.0:1788204910.428375:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204910.428386:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543487104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204910.428403:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204910.428408:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204910.428413:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543487104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204910.428442:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204910.428445:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204910.428450:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204910.428451:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204910.428454:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204910.428457:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204910.428459:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26596 00000400:00000200:3.0:1788204910.428465:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204910.428477:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204910.428480:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204910.428487:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428490:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.428493:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428495:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204910.428498:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204910.428499:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788204910.428501:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204910.428504:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788204910.428507:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788204910.428510:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204910.428513:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428518:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.428521:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428524:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204910.428539:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788204910.428555:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428566:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204910.428570:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204910.428578:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204910.428583:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.428591:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204910.428599:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204910.428610:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.428616:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204910.428624:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788204910.428631:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204910.428641:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204910.428646:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.428653:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204910.428657:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204910.428663:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204910.428672:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411c00. 00000100:00000001:3.0:1788204910.428681:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204910.428694:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543487232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204910.428714:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204910.428719:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204910.428728:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543487232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204910.428749:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204910.428754:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204910.428760:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788204910.428763:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204910.428768:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204910.428782:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204910.428790:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26596 00000400:00000200:3.0:1788204910.428804:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204910.428818:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204910.428828:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204910.428838:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428846:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.428860:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428865:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204910.428876:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204910.428877:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788204910.428880:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204910.428882:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788204910.428893:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788204910.428899:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788204910.428906:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428909:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.428915:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428921:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204910.428924:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788204910.428934:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204910.428947:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204910.428951:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204910.428961:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204910.428969:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204910.429051:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204910.429056:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000100:00000001:3.0:1788204910.429068:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204910.429073:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204910.429081:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.429086:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204910.429134:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788204910.429137:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204910.429141:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204910.429143:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.429145:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204910.429148:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204910.429150:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204910.429155:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412000. 00000100:00000001:3.0:1788204910.429161:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204910.429169:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204910.429189:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204910.429201:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204910.429210:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204910.429228:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204910.429236:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204910.429247:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204910.429249:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204910.429252:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204910.429261:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204910.429271:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26596 00000400:00000200:3.0:1788204910.429288:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204910.429297:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204910.429302:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204910.429310:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429319:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.429325:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429346:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204910.429349:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204910.429351:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788204910.429353:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204910.429355:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788204910.429358:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788204910.429360:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788204910.429364:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429366:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.429368:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429370:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204910.429372:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788204910.429375:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429377:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204910.429379:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204910.429383:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204910.429386:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.429388:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204910.429390:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788204910.429393:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.429394:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204910.429399:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788204910.429401:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204910.429404:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204910.429405:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.429407:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204910.429409:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204910.429410:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204910.429416:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788204910.429420:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204910.429426:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204910.429434:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204910.429437:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204910.429442:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204910.429451:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204910.429453:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204910.429457:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204910.429458:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204910.429459:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204910.429462:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204910.429464:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26596 00000400:00000200:3.0:1788204910.429469:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204910.429473:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204910.429475:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204910.429478:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429480:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.429482:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429484:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204910.429487:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204910.429488:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204910.429489:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204910.429492:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788204910.429494:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788204910.429496:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788204910.429499:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429501:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204910.429503:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429505:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204910.429508:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788204910.429511:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204910.429513:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204910.429516:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204910.429519:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204910.429521:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.429524:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204910.429526:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788204910.429529:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.429530:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204910.429536:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788204910.429538:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204910.429541:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204910.429542:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204910.429544:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204910.429545:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204910.429547:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204910.429551:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788204910.429554:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204910.429559:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204910.429566:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204910.429569:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204910.429572:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204910.429580:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204910.429586:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26606) 00000800:00000200:1.1:1788204910.431172:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204910.431175:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000400:00000001:1.0:1788204910.431263:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204910.431265:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204910.431267:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204910.431268:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204910.431270:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204910.431272:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543487232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204910.431277:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543487232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431282:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204910.431283:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431284:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204910.431285:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204910.431287:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543487232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431291:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204910.431294:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543487232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431298:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204910.431300:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431301:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204910.431302:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431303:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204910.431305:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204910.431306:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204910.431308:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204910.431310:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204910.431312:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543487232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204910.431318:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487232:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204910.431321:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204910.431323:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431342:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204910.431343:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204910.431346:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204910.431347:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204910.431349:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204910.431351:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487360:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204910.431354:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204910.431355:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204910.431357:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204910.431358:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204910.431361:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204910.431362:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204910.431365:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204910.431367:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204910.431369:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204910.431371:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543487360, portal 10 00000100:00000001:1.0:1788204910.431373:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204910.431374:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788204910.431378:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431383:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204910.431384:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204910.431386:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204910.431388:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543487360, offset 0 00000400:00000200:1.0:1788204910.431390:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204910.431393:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204910.431395:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204910.431397:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204910.431400:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204910.431401:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204910.431403:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204910.431405:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471156:471156:256:4294967295] 192.168.201.118@tcp LPNI seq info [471015:471015:8:4294967295] 00000400:00000200:1.0:1788204910.431408:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204910.431413:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204910.431416:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204910.431418:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204910.431420:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204910.431422:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204910.431426:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204910.431428:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788204910.431430:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204910.431431:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204910.431434:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204910.431436:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204910.431438:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204910.431439:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204910.431441:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204910.431443:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431449:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204910.431451:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431455:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.431456:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204910.431459:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204910.431461:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204910.431463:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204910.431464:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321269270 00000100:00000001:1.0:1788204910.431466:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321270534 00000100:00000001:1.0:1788204910.431468:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321269270 00000100:00000001:1.0:1788204910.431469:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321270535 00000100:00000001:1.0:1788204910.431471:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204910.431472:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.431474:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204910.431476:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204910/real 1788204910] req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204910.431481:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204910.431482:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204910.431483:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204910.431484:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204910.431485:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204910.431486:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204910.431488:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204910.431494:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431498:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204910.431499:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431500:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204910.431501:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204910.431503:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431507:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204910.431509:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431513:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204910.431515:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431516:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204910.431517:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431519:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204910.431522:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204910.431525:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204910.431528:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204910.431531:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204910.431544:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204910.431559:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487360:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204910.431564:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204910.431568:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431574:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204910.431575:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204910.431577:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204910.431579:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204910.431580:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204910.431582:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487488:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204910.431585:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204910.431587:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204910.431588:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204910.431590:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204910.431592:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788204910.431594:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204910.431598:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204910.431600:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204910.431604:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204910.431608:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543487488, portal 4 00000100:00000001:1.0:1788204910.431610:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204910.431611:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788204910.431618:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431625:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204910.431628:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204910.431630:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204910.431633:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543487488, offset 0 00000400:00000200:1.0:1788204910.431637:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204910.431642:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204910.431646:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204910.431650:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204910.431653:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204910.431655:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204910.431658:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204910.431661:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471157:471157:256:4294967295] 192.168.201.118@tcp LPNI seq info [471016:471016:8:4294967295] 00000400:00000200:1.0:1788204910.431666:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204910.431673:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204910.431678:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204910.431683:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204910.431687:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204910.431690:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204910.431695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204910.431698:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788204910.431702:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204910.431704:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204910.431708:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204910.431711:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204910.431715:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204910.431717:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204910.431720:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204910.431724:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431733:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204910.431737:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431750:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.431752:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204910.431756:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204910.431758:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204910.431761:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204910.431763:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321269279 00000100:00000001:1.0:1788204910.431765:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321270534 00000100:00000001:1.0:1788204910.431767:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321269279 00000100:00000001:1.0:1788204910.431768:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321270535 00000100:00000001:1.0:1788204910.431770:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204910.431771:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.431773:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204910.431775:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204910/real 1788204910] req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204910.431780:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204910.431781:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204910.431782:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204910.431784:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204910.431785:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204910.431787:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204910.431789:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204910.431794:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431799:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204910.431800:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431801:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204910.431802:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204910.431804:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431809:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204910.431812:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431816:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204910.431818:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431819:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204910.431819:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.431820:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204910.431822:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204910.431823:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204910.431825:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204910.431827:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204910.431829:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204910.431834:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487488:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204910.431836:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204910.431838:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431843:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204910.431844:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204910.431846:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204910.431847:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204910.431848:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204910.431850:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487616:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204910.431852:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204910.431854:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204910.431856:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204910.431858:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204910.431861:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204910.431864:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204910.431867:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204910.431870:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204910.431873:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204910.431876:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543487616, portal 4 00000100:00000001:1.0:1788204910.431878:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204910.431880:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788204910.431887:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431894:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204910.431897:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204910.431899:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204910.431902:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543487616, offset 0 00000400:00000200:1.0:1788204910.431906:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204910.431911:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204910.431914:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204910.431916:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204910.431918:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204910.431919:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204910.431921:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204910.431924:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471158:471158:256:4294967295] 192.168.201.118@tcp LPNI seq info [471017:471017:8:4294967295] 00000400:00000200:1.0:1788204910.431927:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204910.431931:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204910.431935:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204910.431937:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204910.431940:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204910.431942:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204910.431945:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204910.431947:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788204910.431950:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204910.431951:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204910.431954:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204910.431956:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204910.431958:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204910.431959:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204910.431961:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204910.431964:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431969:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204910.431971:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204935 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.431976:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.431977:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204910.431979:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204910.431982:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204910.431984:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204910.431985:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321269302 00000100:00000001:1.0:1788204910.431987:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321270534 00000100:00000001:1.0:1788204910.431989:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321269302 00000100:00000001:1.0:1788204910.431991:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321270535 00000100:00000001:1.0:1788204910.431992:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204910.431993:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.431994:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204910.431996:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204910/real 1788204910] req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204910.432002:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204910.432003:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204910.432004:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204910.432006:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204910.432007:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204910.432009:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204910.432012:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204910.432021:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.432028:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204910.432029:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432031:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204910.432033:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204910.432037:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.432044:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204910.432048:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.432056:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204910.432059:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432061:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204910.432062:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432064:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204910.432066:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204910.432069:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204910.432073:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204910.432076:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204910.432080:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204910.432086:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487616:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204910.432088:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432106:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204910.432109:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543487232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.432114:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204910.432116:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204910.432117:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204910.432119:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204910.432121:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204910.432122:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204910.432123:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204910.432124:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788204910.432127:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204910.432128:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204910.432129:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788204910.432131:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204910.432132:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432134:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204910.432136:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543487360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.432140:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204910.432141:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204910.432142:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204910.432143:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204910.432144:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204910.432145:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204910.432146:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204910.432147:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788204910.432148:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204910.432149:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204910.432150:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788204910.432151:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204910.432152:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432153:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204910.432155:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543487488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.432158:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204910.432160:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204910.432161:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788204910.432162:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204910.432163:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204910.432164:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204910.432165:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204910.432166:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788204910.432167:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204910.432168:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204910.432169:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788204910.432171:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204910.432171:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432172:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204910.432174:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543487616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204935 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204910.432178:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204910.432178:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204910.432179:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204910.432180:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204910.432181:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204910.432182:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204910.432183:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204910.432184:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788204910.432185:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204910.432185:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204910.432186:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788204910.432187:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204910.432188:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432189:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204910.432192:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204910.432193:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204910.432194:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204910.432195:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204911.260238:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204911.260238:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204911.260247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204911.260248:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204911.260249:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204911.260255:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204911.260255:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204911.260263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204911.260266:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204911.260266:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204911.260275:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204911.260275:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204911.260279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204911.260283:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204911.260284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204911.260284:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.260292:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204911.260293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204911.260296:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204911.260297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204911.260300:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204911.260305:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204911.260306:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204911.260307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204911.260309:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204911.260311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204911.260505:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204911.260512:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204911.260518:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204911.772219:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321270535, at 4321270592 00000100:00000010:3.0:1788204911.772237:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16478a0. 00000100:00000040:2.0:1788204911.772349:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321270534, at 4321270592 00000100:00000010:2.0:1788204911.772365:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00000100:00000001:3.0:1788204912.284233:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204912.284241:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204912.284243:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204912.284248:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204912.284253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204912.284253:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204912.284258:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204912.284260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204912.284260:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204912.284263:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204912.284265:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204912.284278:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1788204912.284279:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204912.284282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204912.284284:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204912.284288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204912.284289:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204912.284291:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204912.284312:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204912.284316:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204912.284318:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204912.284321:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204912.284324:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204912.284330:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204912.284330:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204912.284334:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204912.284412:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204912.284417:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204912.284421:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204913.308197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204913.308197:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204913.308202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204913.308205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204913.308205:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204913.308208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204913.308210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.308210:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204913.308217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204913.308217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204913.308217:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204913.308221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204913.308224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204913.308224:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204913.308228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204913.308229:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.308229:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204913.308231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204913.308235:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204913.308235:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204913.308237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204913.308240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204913.308240:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204913.308241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204913.308247:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204913.308254:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204913.308270:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204913.308274:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204913.308292:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204914.332171:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204914.332177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204914.332179:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204914.332181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204914.332182:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204914.332186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204914.332188:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204914.332189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204914.332192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204914.332194:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788204914.332194:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204914.332195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332200:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1788204914.332200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204914.332200:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204914.332205:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204914.332207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204914.332208:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204914.332210:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788204914.332213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204914.332213:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204914.332214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204914.332219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204914.332220:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204914.332220:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204914.332223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204914.332228:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204914.332228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204914.332249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204914.332251:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204914.332253:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332256:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204914.332258:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332289:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332292:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332297:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204914.332299:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204914.332301:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204914.332302:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332304:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204914.332306:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332308:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204914.332309:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204914.332366:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204914.332371:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204914.332375:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788204915.356239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204915.356239:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204915.356239:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204915.356264:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204915.356265:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204915.356271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.356272:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204915.356273:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204915.356286:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204915.356287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204915.356287:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204915.356294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.356299:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204915.356300:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204915.356301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204915.356305:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.356307:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204915.356307:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.356320:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204915.356321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204915.356322:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204915.356325:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.356326:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204915.356329:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.356336:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204915.356336:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204915.356400:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204915.356406:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204915.356410:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204915.548218:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204915.548227:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204915.548229:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204915.548232:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204915.548236:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204915.548257:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26601 00000400:00000200:2.0:1788204915.548266:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204915.548272:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204915.548275:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204915.548278:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548281:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204915.548282:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548284:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204915.548287:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204915.548288:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204915.548291:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204915.548294:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204915.548296:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204915.548298:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204915.548300:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548301:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204915.548302:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548303:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204915.548306:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204915.548309:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548311:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204915.548313:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204915.548315:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204915.548317:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.548319:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204915.548321:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204915.548329:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204915.548335:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204915.548338:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204915.548339:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.548342:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204915.548344:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204915.548348:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204915.548354:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788204915.548360:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204915.548368:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204915.548376:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204915.548380:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204915.548383:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204915.548402:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204915.548404:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204915.548407:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204915.548408:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204915.548409:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204915.548411:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204915.548413:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26601 00000400:00000200:2.0:1788204915.548416:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204915.548418:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204915.548420:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204915.548423:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548425:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204915.548427:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548428:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204915.548430:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204915.548431:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204915.548434:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204915.548436:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204915.548438:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204915.548439:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204915.548443:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548445:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204915.548447:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548448:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204915.548450:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204915.548452:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204915.548456:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204915.548456:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204915.548459:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788204915.548464:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788204915.548467:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204915.548469:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788204915.548471:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.548475:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204915.548478:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1788204915.548484:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788204915.548484:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.548491:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204915.548505:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000100:00000001:1.0:1788204915.548507:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204915.548512:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204915.548513:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788204915.548517:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204915.548519:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204915.548521:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204915.548526:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204915.548530:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543487744:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788204915.548534:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.548535:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204915.548540:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788204915.548540:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204915.548544:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204915.548546:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204915.548548:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788204915.548551:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204915.548558:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000010:2.0:1788204915.548560:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788204915.548562:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.548571:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204915.548572:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204915.548580:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000100:00100000:2.0:1788204915.548587:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1788204915.548589:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204915.548602:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543487744, portal 25 00000100:00000001:2.0:1788204915.548606:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204915.548607:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204915.548612:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 00000100:00000001:2.0:1788204915.548616:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204915.548622:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204915.548630:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.548633:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204915.548641:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204915.548644:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204915.548652:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543487744, offset 0 00000400:00000200:1.0:1788204915.548661:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548663:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204915.548669:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788204915.548671:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:2.0:1788204915.548680:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204915.548682:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1788204915.548684:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00080000:2.0:1788204915.548690:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788204915.548693:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788204915.548701:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788204915.548705:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00080000:2.0:1788204915.548712:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26601 00000400:00000200:1.0:1788204915.548719:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788204915.548726:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.548728:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:2.0:1788204915.548735:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:1.0:1788204915.548745:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471159:471159:256:4294967295] 192.168.201.118@tcp LPNI seq info [471018:471018:8:4294967295] 00000100:00080000:2.0:1788204915.548745:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204915.548759:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548763:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788204915.548768:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788204915.548769:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548779:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204915.548782:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204915.548783:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204915.548784:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204915.548786:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204915.548789:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788204915.548797:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000020:00000001:2.0:1788204915.548801:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204915.548807:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548811:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1788204915.548816:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000040:2.0:1788204915.548817:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.548825:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204915.548828:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204915.548831:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000010:1.0:1788204915.548832:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000100:00000001:2.0:1788204915.548833:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204915.548837:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204915.548842:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204915.548845:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.548849:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.0:1788204915.548853:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000008:00100000:2.0:1788204915.548853:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204915.548864:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.548866:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788204915.548867:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000100:00000010:2.0:1788204915.548870:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204915.548875:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204915.548879:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204915.548881:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.548886:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1788204915.548888:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:2.0:1788204915.548889:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204915.548893:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204915.548899:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000800:00000200:1.1:1788204915.548899:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000100:00000001:2.0:1788204915.548906:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204915.548916:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788204915.548918:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.548925:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000100:00000001:2.0:1788204915.548927:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204915.548931:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204915.548936:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204915.548938:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788204915.548952:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204915.548957:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1788204915.548957:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204915.548972:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204915.548974:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204915.548975:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788204915.548976:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204915.548985:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:2.0:1788204915.548985:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204915.548996:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26601 00000100:00000001:1.0:1788204915.548997:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1788204915.549011:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:1.0:1788204915.549013:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788204915.549020:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204915.549028:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204915.549038:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204915.549039:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.549046:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204915.549050:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204915.549053:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.549065:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204915.549073:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204915.549081:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000100:00000001:1.0:1788204915.549082:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204915.549119:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204915.549122:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204915.549125:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204915.549127:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204915.549130:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.549131:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204915.549133:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.549134:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204915.549136:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000001:1.0:1788204915.549139:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1788204915.549139:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204915.549146:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204915.549148:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788204915.549152:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204915.549152:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204915.549164:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.549167:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788204915.549171:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00100000:2.0:1788204915.549171:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:1.0:1788204915.549177:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321275654 00000008:00000001:2.0:1788204915.549178:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.549185:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204915.549189:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000100:00000010:1.0:1788204915.549190:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647ba0. 00000020:00000040:2.0:1788204915.549203:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788204915.549207:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 02000000:00000001:2.0:1788204915.549208:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204915.549215:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.549216:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204915.549217:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204915.549219:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204915.549223:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:1.0:1788204915.549225:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321275654 00000100:00000001:2.0:1788204915.549226:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204915.549232:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647e40. 00000100:00100000:2.0:1788204915.549236:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204915.549250:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788204915.549261:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204915.549263:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204915.549265:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.549267:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.549271:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1788204915.549275:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204915.549281:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204915/real 1788204915] req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204915.549321:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204915.549324:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204915.549330:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204915.549332:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:1.0:1788204915.549334:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788204915.549335:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1788204915.549337:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204915.549343:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000100:00000001:2.0:1788204915.549344:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204915.549352:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26601 00000400:00000200:1.0:1788204915.549353:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204915.549364:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000400:00000200:2.0:1788204915.549372:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204915.549374:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1788204915.549379:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000200:1.0:1788204915.549386:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204915.549387:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204915.549403:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.549413:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204915.549422:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000200:1.0:1788204915.549424:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204915.549431:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204915.549445:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:1.0:1788204915.549447:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1788204915.549449:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000001:1.0:1788204915.549451:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788204915.549455:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788204915.549457:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1788204915.549459:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788204915.549460:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788204915.549469:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00100000:1.0:1788204915.549471:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204915.549478:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788204915.549481:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204915.549495:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204915.549496:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204915.549511:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204915.549512:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204915.549517:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.549523:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1788204915.549525:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204915.549526:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.549533:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204915.549535:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204915.549536:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204915.549542:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:2.0:1788204915.549544:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788204915.549549:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204915.549552:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1788204915.549556:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000040:1.0:1788204915.549564:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204915.549564:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204915.549572:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204915.549580:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788204915.549586:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543487744:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788204915.549587:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.549595:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204915.549602:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788204915.549603:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788204915.549611:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788204915.549612:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204915.549619:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543487744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204915.549620:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204915.549631:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204915.549638:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788204915.549639:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204915.549642:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204915.549649:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:2.0:1788204915.549649:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204915.549658:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.549659:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204915.549662:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000001:2.0:1788204915.549664:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000020:00000040:1.0:1788204915.549665:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204915.549672:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000001:2.0:1788204915.549672:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204915.549682:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204915.549683:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:1.0:1788204915.549700:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:2.0:1788204915.549701:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 02000000:00000001:1.0:1788204915.549703:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204915.549711:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:2.0:1788204915.549713:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.549715:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204915.549718:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788204915.549722:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204915.549723:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.549743:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204915.549748:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204915.549748:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204915.549759:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204915.549760:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.549764:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204915.549768:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788204915.549770:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204915.549777:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204915.549796:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204915.549805:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26612) 00000100:00000001:1.0:1788204915.549806:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204915.549810:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.549813:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204915.549817:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204915.549819:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204915.549824:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487872:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204915.549830:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204915.549833:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204915.549834:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.549836:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204915.549841:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204915.549843:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204915.549848:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204915.549851:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204915.549856:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204915.549860:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543487872, portal 10 00000100:00000001:1.0:1788204915.549862:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204915.549864:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000040:1.0:1788204915.549872:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.549880:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204915.549882:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204915.549885:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204915.549888:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543487872, offset 0 00000400:00000200:1.0:1788204915.549891:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204915.549895:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.549899:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204915.549903:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204915.549906:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204915.549909:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204915.549912:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204915.549916:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471160:471160:256:4294967295] 192.168.201.118@tcp LPNI seq info [471019:471019:8:4294967295] 00000400:00000200:1.0:1788204915.549920:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.549928:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204915.549933:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.549937:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204915.549941:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204915.549945:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204915.549950:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204915.549953:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1788204915.549957:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.549959:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204915.549963:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204915.549967:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204915.549970:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204915.549972:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204915.549975:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204915.549978:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.549985:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204915.549987:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.549992:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.549993:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204915.549995:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204915.549999:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788204915.550001:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204915.550003:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321274387 00000100:00000001:1.0:1788204915.550006:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321275654 00000100:00000001:1.0:1788204915.550008:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321274387 00000100:00000001:1.0:1788204915.550009:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321275654 00000100:00000001:1.0:1788204915.550012:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204915.550013:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550015:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204915.550017:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204915/real 1788204915] req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204915.550022:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204915.550023:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204915.550024:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204915.550026:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204915.550027:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204915.550028:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204915.550030:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204915.550036:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550040:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204915.550041:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550042:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204915.550043:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204915.550045:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550049:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204915.550051:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550055:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204915.550057:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550058:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204915.550058:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550060:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204915.550061:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204915.550064:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204915.550066:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204915.550068:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204915.550071:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204915.550076:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543487872:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204915.550078:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204915.550081:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550085:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204915.550086:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.550087:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204915.550106:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204915.550108:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204915.550111:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488000:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204915.550113:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204915.550115:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204915.550116:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.550118:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204915.550120:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788204915.550122:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204915.550124:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204915.550126:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204915.550128:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204915.550131:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543488000, portal 10 00000100:00000001:1.0:1788204915.550132:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204915.550134:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000100:00000040:1.0:1788204915.550138:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550147:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204915.550149:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204915.550150:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204915.550152:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543488000, offset 0 00000400:00000200:1.0:1788204915.550154:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204915.550157:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.550159:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204915.550161:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204915.550163:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204915.550165:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204915.550167:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204915.550169:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471161:471161:256:4294967295] 192.168.201.118@tcp LPNI seq info [471020:471020:8:4294967295] 00000400:00000200:1.0:1788204915.550172:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.550176:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204915.550180:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.550182:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204915.550185:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204915.550187:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204915.550190:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204915.550192:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1788204915.550195:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.550196:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204915.550198:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204915.550201:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204915.550203:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204915.550204:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204915.550206:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204915.550208:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550214:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204915.550216:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550225:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550226:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204915.550228:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204915.550230:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204915.550232:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204915.550234:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321274389 00000100:00000001:1.0:1788204915.550236:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321275654 00000100:00000001:1.0:1788204915.550237:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321274389 00000100:00000001:1.0:1788204915.550255:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321275654 00000100:00000001:1.0:1788204915.550257:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204915.550259:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550261:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204915.550264:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204915/real 1788204915] req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204915.550270:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204915.550271:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204915.550272:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204915.550274:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204915.550275:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204915.550277:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204915.550279:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204915.550285:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550289:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204915.550290:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550291:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204915.550292:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204915.550294:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550299:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204915.550302:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550306:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204915.550307:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550308:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204915.550309:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550310:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204915.550312:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204915.550314:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204915.550316:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204915.550317:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204915.550320:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204915.550324:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488000:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204915.550327:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204915.550330:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550335:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204915.550336:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.550337:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204915.550339:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204915.550340:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204915.550342:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488128:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204915.550344:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204915.550346:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204915.550348:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.550349:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204915.550351:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204915.550353:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204915.550355:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204915.550357:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204915.550360:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204915.550362:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543488128, portal 4 00000100:00000001:1.0:1788204915.550363:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204915.550364:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:1.0:1788204915.550368:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550373:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204915.550375:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204915.550376:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204915.550377:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543488128, offset 0 00000400:00000200:1.0:1788204915.550380:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204915.550382:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.550385:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204915.550386:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204915.550388:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204915.550389:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204915.550391:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204915.550393:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471162:471162:256:4294967295] 192.168.201.118@tcp LPNI seq info [471021:471021:8:4294967295] 00000400:00000200:1.0:1788204915.550396:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.550401:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204915.550404:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.550406:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204915.550409:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204915.550411:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204915.550414:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204915.550416:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1788204915.550418:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.550420:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204915.550422:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204915.550424:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204915.550427:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204915.550428:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204915.550429:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204915.550431:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550437:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204915.550439:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550443:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550444:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204915.550446:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204915.550449:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204915.550451:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204915.550453:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321274398 00000100:00000001:1.0:1788204915.550455:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321275654 00000100:00000001:1.0:1788204915.550456:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321274398 00000100:00000001:1.0:1788204915.550458:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321275654 00000100:00000001:1.0:1788204915.550459:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204915.550460:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550462:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204915.550464:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204915/real 1788204915] req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204915.550468:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204915.550469:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204915.550470:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204915.550471:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204915.550472:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204915.550474:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204915.550476:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204915.550481:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550485:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204915.550486:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550487:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204915.550488:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204915.550490:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550494:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204915.550496:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550500:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204915.550501:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550502:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204915.550503:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550504:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204915.550506:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204915.550507:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204915.550509:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204915.550510:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204915.550513:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204915.550517:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488128:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204915.550520:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550521:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204915.550524:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543487872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550529:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204915.550530:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204915.550531:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204915.550533:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204915.550534:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204915.550534:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204915.550536:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204915.550537:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788204915.550540:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204915.550541:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204915.550542:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:1.0:1788204915.550544:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204915.550545:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550546:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204915.550548:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543488000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550556:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204915.550557:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204915.550559:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788204915.550561:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204915.550562:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204915.550564:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204915.550566:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204915.550567:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:1.0:1788204915.550571:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204915.550573:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204915.550574:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:1.0:1788204915.550578:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204915.550579:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550581:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204915.550585:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543488128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550593:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204915.550595:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204915.550596:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204915.550597:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204915.550598:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204915.550599:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204915.550601:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204915.550602:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d000. 02000000:00000001:1.0:1788204915.550603:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204915.550604:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204915.550605:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788204915.550606:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204915.550607:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550608:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550612:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204915.550613:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550614:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204915.550615:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204915.550617:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204915.550620:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550625:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204915.550626:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.550628:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204915.550629:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204915.550630:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204915.550632:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488256:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204915.550635:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204915.550636:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204915.550638:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204915.550639:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204915.550641:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204915.550642:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204915.550645:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204915.550647:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204915.550649:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204915.550651:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543488256, portal 4 00000100:00000001:1.0:1788204915.550653:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204915.550654:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922048 : -109424377629568 : ffff9c7aa6d77480) 00000100:00000040:1.0:1788204915.550659:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550663:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204915.550665:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204915.550666:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204915.550668:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543488256, offset 0 00000400:00000200:1.0:1788204915.550670:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204915.550673:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.550675:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204915.550677:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204915.550679:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204915.550681:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204915.550683:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204915.550685:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471163:471163:256:4294967295] 192.168.201.118@tcp LPNI seq info [471022:471022:8:4294967295] 00000400:00000200:1.0:1788204915.550688:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204915.550692:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204915.550695:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.550697:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204915.550700:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204915.550702:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1788204915.550707:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204915.550710:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1788204915.550714:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204915.550716:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204915.550719:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204915.550723:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204915.550726:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204915.550728:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204915.550731:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204915.550735:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550749:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204915.550752:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204940 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550758:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550760:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204915.550763:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204915.550767:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204915.550770:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204915.550772:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321274421 00000100:00000001:1.0:1788204915.550775:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321275654 00000100:00000001:1.0:1788204915.550778:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321274421 00000100:00000001:1.0:1788204915.550780:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321275654 00000100:00000001:1.0:1788204915.550782:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204915.550784:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550786:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204915.550791:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204915/real 1788204915] req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204915.550799:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204915.550800:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204915.550802:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204915.550805:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204915.550806:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204915.550809:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204915.550813:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204915.550822:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550829:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204915.550831:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550833:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204915.550835:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204915.550838:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550845:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204915.550849:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550857:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204915.550860:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550861:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204915.550863:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550865:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204915.550867:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204915.550870:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204915.550874:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204915.550877:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204915.550881:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204915.550890:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488256:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204915.550895:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550896:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204915.550900:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77480 x1875055543488256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204940 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204915.550907:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204915.550909:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204915.550911:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204915.550914:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204915.550916:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204915.550918:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204915.550920:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204915.550922:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e800. 02000000:00000001:1.0:1788204915.550925:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204915.550926:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204915.550928:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77480. 00000100:00000001:1.0:1788204915.550930:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204915.550932:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550934:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204915.550938:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204915.550939:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204915.550941:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204915.550943:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204916.380186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204916.380186:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204916.380192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204916.380194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204916.380194:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204916.380198:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204916.380199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204916.380199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204916.380204:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204916.380205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204916.380208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204916.380209:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204916.380210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204916.380211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204916.380214:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204916.380215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204916.380217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204916.380222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204916.380222:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204916.380223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204916.380240:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204916.380242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204916.380246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204916.380247:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204916.380251:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204916.380255:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204916.380268:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204916.380271:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204916.380273:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204916.892246:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321275654, at 4321275712 00000100:00000010:3.0:1788204916.892266:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00000100:00000040:2.0:1788204916.893327:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321275654, at 4321275712 00000100:00000010:2.0:1788204916.893336:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647ba0. 00000100:00000001:1.0:1788204917.404228:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204917.404229:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204917.404237:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204917.404237:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204917.404240:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204917.404240:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204917.404246:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204917.404246:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204917.404250:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204917.404250:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204917.404253:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204917.404254:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204917.404257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204917.404257:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204917.404260:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204917.404260:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204917.404264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204917.404264:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204917.404321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204917.404327:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204917.404334:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204917.404335:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204917.404338:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204917.404340:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204917.404343:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204917.404344:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204917.404414:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204917.404419:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204917.404424:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204918.429414:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204918.429415:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204918.429423:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204918.429423:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204918.429427:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204918.429427:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204918.429434:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204918.429434:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204918.429439:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204918.429440:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204918.429445:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204918.429445:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204918.429452:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204918.429452:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204918.429456:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204918.429459:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204918.429848:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204918.429852:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204918.429868:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204918.429869:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204918.429873:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204918.429874:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204918.429877:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204918.429878:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204918.429927:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204918.429937:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204918.429939:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204918.430630:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204918.430635:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204919.452189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204919.452189:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204919.452197:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204919.452198:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204919.452201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.452201:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204919.452205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204919.452206:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204919.452208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204919.452209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204919.452211:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204919.452212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204919.452215:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204919.452216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204919.452217:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204919.452220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204919.452221:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204919.452223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204919.452226:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204919.452230:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204919.452245:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204919.452257:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204919.452260:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204919.452267:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204919.452269:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204919.452270:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204919.452274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204919.452275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204919.452276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204919.452277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204919.452279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204919.452280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788204919.452299:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788204919.452301:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788204919.452302:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204920.476213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204920.476213:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204920.476217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204920.476220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204920.476221:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204920.476223:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204920.476224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204920.476225:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204920.476230:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204920.476231:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204920.476233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204920.476234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204920.476237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204920.476237:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204920.476238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.476241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204920.476242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204920.476243:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204920.476244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.476244:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204920.476248:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204920.476250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204920.476251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204920.476252:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204920.476255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204920.476257:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204920.476321:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204920.476325:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204920.476328:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204920.669370:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204920.669388:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204920.669390:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204920.669393:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204920.669397:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204920.669408:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26607 00000400:00000200:2.0:1788204920.669416:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204920.670486:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204920.670492:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204920.670496:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670499:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204920.670501:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670504:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204920.670508:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204920.670510:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204920.670524:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204920.670529:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204920.670531:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204920.670534:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204920.670536:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670538:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204920.670540:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670542:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204920.670545:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204920.670548:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670551:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204920.670553:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204920.670557:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204920.670560:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204920.670563:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204920.670565:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204920.670575:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204920.670579:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204920.670582:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204920.670584:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204920.670587:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204920.670590:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204920.670594:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204920.670603:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788204920.670610:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204920.670618:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204920.670628:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204920.670632:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204920.670636:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204920.670658:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204920.670661:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204920.670664:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204920.670665:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204920.670666:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204920.670668:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204920.670670:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26607 00000400:00000200:2.0:1788204920.670674:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204920.670677:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204920.670678:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204920.670680:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670682:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204920.670683:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670684:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204920.670686:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204920.670687:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204920.670688:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204920.670689:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204920.670691:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204920.670692:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204920.670694:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670695:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204920.670696:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670698:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204920.670699:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204920.670701:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670703:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204920.670705:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204920.670710:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204920.670712:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.670713:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204920.670713:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204920.670717:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788204920.670720:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1788204920.670720:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.670724:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204920.670724:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204920.670726:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788204920.670730:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204920.670733:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204920.670734:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204920.670735:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204920.670736:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204920.670738:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204920.670740:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204920.670744:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788204920.670750:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.670752:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204920.670756:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204920.670758:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788204920.670761:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204920.670765:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204920.670771:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788204920.670772:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204920.670776:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204920.670780:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543488384:192.168.201.118@tcp:250:kworker.0 00000100:00000040:2.0:1788204920.670785:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.670787:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204920.670791:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204920.670794:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.670798:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788204920.670802:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204920.670804:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8400. 00000100:00080000:2.0:1788204920.670806:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788204920.670808:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204920.670811:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204920.670813:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204920.670814:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788204920.670816:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204920.670818:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204920.670821:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26607 00000400:00000200:2.0:1788204920.670824:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204920.670827:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000010:1.0:1788204920.670829:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000100:00080000:2.0:1788204920.670829:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204920.670834:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670836:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1788204920.670838:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000040:2.0:1788204920.670838:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670843:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204920.670844:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204920.670845:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204920.670846:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:1.0:1788204920.670848:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543488384, portal 25 00000020:00000040:2.0:1788204920.670848:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000001:1.0:1788204920.670852:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1788204920.670853:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788204920.670855:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331914880 : -109424377636736 : ffff9c7aa6d75880) 00000020:00000001:2.0:1788204920.670856:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204920.670860:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:1.0:1788204920.670863:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204920.670864:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204920.670868:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670872:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204920.670877:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000001:1.0:1788204920.670878:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788204920.670881:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000040:1.0:1788204920.670886:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670886:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204920.670889:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 02000000:00000001:2.0:1788204920.670890:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204920.670893:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1788204920.670894:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543488384, offset 0 02000000:00000001:2.0:1788204920.670898:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204920.670900:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788204920.670903:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204920.670910:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00100000:2.0:1788204920.670914:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1788204920.670916:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000008:00000001:2.0:1788204920.670919:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.670922:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788204920.670922:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.670928:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000010:2.0:1788204920.670929:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000400:00000200:1.0:1788204920.670932:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1788204920.670934:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000200:1.0:1788204920.670935:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 02000000:00000001:2.0:1788204920.670941:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1788204920.670942:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471164:471164:256:4294967295] 192.168.201.118@tcp LPNI seq info [471023:471023:8:4294967295] 02000000:00000001:2.0:1788204920.670944:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.670949:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204920.670950:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204920.670953:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204920.670956:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.670958:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000010:2.0:1788204920.670962:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000800:00000200:1.0:1788204920.670965:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204920.670969:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1788204920.670971:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204920.670979:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00100000:2.0:1788204920.670980:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1788204920.670983:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204920.670992:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204920.670997:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000100:00000001:2.0:1788204920.670999:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000100:1.0:1788204920.671002:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671004:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1788204920.671005:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204920.671012:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1788204920.671012:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204920.671022:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788204920.671030:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.671031:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000100:00080000:2.0:1788204920.671034:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1788204920.671035:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00080000:2.0:1788204920.671039:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204920.671041:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204920.671043:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1788204920.671043:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204920.671049:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1788204920.671052:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204920.671055:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26607 00000400:00000200:2.0:1788204920.671065:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204920.671072:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1788204920.671072:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204920.671077:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204920.671079:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204920.671083:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671088:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204920.671109:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671112:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204920.671114:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204920.671116:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204920.671118:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204920.671120:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204920.671123:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204920.671125:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204920.671128:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671130:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204920.671134:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671138:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204920.671142:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204920.671142:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000001:1.0:1788204920.671148:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1788204920.671149:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671170:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204920.671172:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204920.671174:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204920.671176:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204920.671178:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204920.671181:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204920.671183:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.671185:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204920.671186:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.671193:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:2.0:1788204920.671193:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204920.671198:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788204920.671199:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321280776 02000000:00000001:2.0:1788204920.671203:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204920.671206:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204920.671207:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647600. 00000100:00000001:2.0:1788204920.671209:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204920.671212:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788204920.671213:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 02000000:00000001:2.0:1788204920.671215:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.671221:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321280776 02000000:00000010:2.0:1788204920.671222:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000010:1.0:1788204920.671226:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647720. 00000100:00000001:1.0:1788204920.671229:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1788204920.671229:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.671234:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788204920.671237:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204920.671238:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.671242:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788204920.671247:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1788204920.671248:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204920/real 1788204920] req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204920.671251:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204920.671259:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204920.671263:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204920.671266:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788204920.671267:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204920.671269:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000100:00080000:2.0:1788204920.671271:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788204920.671272:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00080000:2.0:1788204920.671274:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000010:1.0:1788204920.671275:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:2.0:1788204920.671277:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204920.671278:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1788204920.671282:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1788204920.671284:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204920.671288:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1788204920.671292:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204920.671295:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26607 00000100:00000001:1.0:1788204920.671304:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204920.671306:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204920.671307:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204920.671309:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.671312:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788204920.671315:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00100000:1.0:1788204920.671317:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204920.671321:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204920.671325:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204920.671328:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671331:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204920.671332:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204920.671335:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671340:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204920.671345:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:1.0:1788204920.671346:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1788204920.671347:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000001:1.0:1788204920.671350:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788204920.671351:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788204920.671354:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1788204920.671354:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788204920.671356:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788204920.671358:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788204920.671359:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:2.0:1788204920.671360:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788204920.671361:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204920.671366:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1788204920.671366:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671368:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1788204920.671372:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000040:2.0:1788204920.671372:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671376:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204920.671377:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:2.0:1788204920.671379:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000040:1.0:1788204920.671383:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204920.671384:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204920.671389:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204920.671393:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204920.671396:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788204920.671399:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543488384:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788204920.671400:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204920.671402:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204920.671406:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788204920.671407:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788204920.671409:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.671410:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788204920.671413:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204920.671414:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75880 x1875055543488384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204920.671422:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788204920.671423:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204920.671426:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:2.0:1788204920.671428:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000010:1.0:1788204920.671429:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:2.0:1788204920.671433:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204920.671434:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204920.671438:0:256132:0:(genops.c:1239:class_import_put()) Process entered 02000000:00000001:2.0:1788204920.671438:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788204920.671440:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204920.671443:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000100:00000001:2.0:1788204920.671443:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:1.0:1788204920.671448:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e800. 02000000:00000001:2.0:1788204920.671448:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204920.671452:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.671453:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204920.671456:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204920.671458:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75880. 00000100:00000001:1.0:1788204920.671461:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000010:2.0:1788204920.671461:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:1.0:1788204920.671463:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.671466:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204920.671468:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204920.671476:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204920.671483:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204920.671485:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204920.671488:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204920.671495:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204920.671499:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26617) 00000100:00000001:1.0:1788204920.672516:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204920.672519:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.672523:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204920.672525:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204920.672527:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204920.672534:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672543:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204920.672544:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.672547:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204920.672549:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204920.672551:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204920.672554:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488512:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204920.672557:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204920.672560:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204920.672561:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.672563:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204920.672566:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204920.672568:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.672570:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204920.672572:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204920.672576:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204920.672579:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543488512, portal 10 00000100:00000001:1.0:1788204920.672581:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204920.672581:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331918464 : -109424377633152 : ffff9c7aa6d76680) 00000100:00000040:1.0:1788204920.672586:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672590:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204920.672592:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204920.672593:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204920.672595:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543488512, offset 0 00000400:00000200:1.0:1788204920.672598:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204920.672602:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204920.672605:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204920.672608:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204920.672611:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204920.672613:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204920.672615:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204920.672618:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471165:471165:256:4294967295] 192.168.201.118@tcp LPNI seq info [471024:471024:8:4294967295] 00000400:00000200:1.0:1788204920.672622:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204920.672630:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204920.672634:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204920.672638:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204920.672643:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204920.672646:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204920.672651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204920.672653:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1788204920.672657:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204920.672660:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204920.672664:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204920.672667:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204920.672671:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204920.672673:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204920.672676:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204920.672679:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672693:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204920.672696:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672703:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.672705:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204920.672710:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204920.672715:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788204920.672718:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204920.672721:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321279510 00000100:00000001:1.0:1788204920.672724:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321280776 00000100:00000001:1.0:1788204920.672726:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321279510 00000100:00000001:1.0:1788204920.672728:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321280776 00000100:00000001:1.0:1788204920.672730:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204920.672732:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.672734:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204920.672738:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204920/real 1788204920] req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204920.672746:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204920.672748:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204920.672750:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204920.672752:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204920.672754:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204920.672756:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204920.672759:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204920.672767:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672773:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204920.672775:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.672777:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.672778:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204920.672782:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672788:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204920.672791:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672797:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204920.672800:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.672801:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.672802:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.672804:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204920.672806:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204920.672809:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204920.672813:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204920.672815:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204920.672819:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204920.672828:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488512:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204920.672832:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204920.672835:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672841:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204920.672842:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.672845:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204920.672846:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204920.672848:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204920.672852:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488640:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204920.672854:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204920.672855:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204920.672857:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.672858:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204920.672860:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788204920.672861:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.672863:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204920.672865:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204920.672867:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204920.672869:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543488640, portal 10 00000100:00000001:1.0:1788204920.672870:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204920.672871:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331915776 : -109424377635840 : ffff9c7aa6d75c00) 00000100:00000040:1.0:1788204920.672875:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672878:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204920.672880:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204920.672881:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204920.672882:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543488640, offset 0 00000400:00000200:1.0:1788204920.672884:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204920.672887:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204920.672889:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204920.672892:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204920.672894:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204920.672896:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204920.672898:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204920.672901:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471166:471166:256:4294967295] 192.168.201.118@tcp LPNI seq info [471025:471025:8:4294967295] 00000400:00000200:1.0:1788204920.672905:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204920.672911:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204920.672914:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204920.672917:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204920.672921:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204920.672924:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204920.672928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204920.672930:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1788204920.672934:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204920.672936:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204920.672939:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204920.672942:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204920.672945:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204920.672946:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204920.672948:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204920.672950:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672955:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204920.672957:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672961:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.672961:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204920.672964:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204920.672966:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204920.672968:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204920.672969:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321279511 00000100:00000001:1.0:1788204920.672971:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321280776 00000100:00000001:1.0:1788204920.672972:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321279511 00000100:00000001:1.0:1788204920.672973:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321280776 00000100:00000001:1.0:1788204920.672975:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204920.672976:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.672977:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204920.672979:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204920/real 1788204920] req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204920.672984:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204920.672985:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204920.672986:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204920.672987:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204920.672988:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204920.672989:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204920.672991:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204920.672995:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.672999:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204920.672999:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.673000:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.673001:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204920.673003:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673006:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204920.673008:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673012:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204920.673014:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.673015:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.673015:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.673016:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204920.673018:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204920.673019:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204920.673021:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204920.673022:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204920.673024:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204920.673032:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488640:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204920.673035:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204920.673037:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673040:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204920.673041:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.673042:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204920.673043:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204920.673044:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204920.673046:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488768:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204920.673048:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204920.673049:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204920.673050:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.673051:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204920.673053:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204920.673054:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.673056:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204920.673058:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204920.673060:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204920.673061:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543488768, portal 4 00000100:00000001:1.0:1788204920.673062:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204920.673063:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922048 : -109424377629568 : ffff9c7aa6d77480) 00000100:00000040:1.0:1788204920.673066:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673070:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204920.673071:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204920.673073:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204920.673074:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543488768, offset 0 00000400:00000200:1.0:1788204920.673076:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204920.673078:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204920.673080:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204920.673082:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204920.673084:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204920.673086:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204920.673087:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204920.673102:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471167:471167:256:4294967295] 192.168.201.118@tcp LPNI seq info [471026:471026:8:4294967295] 00000400:00000200:1.0:1788204920.673106:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204920.673109:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204920.673112:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204920.673114:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204920.673116:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204920.673119:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204920.673121:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204920.673123:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1788204920.673125:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204920.673126:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204920.673128:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204920.673130:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204920.673132:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204920.673133:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204920.673134:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204920.673136:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673141:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204920.673143:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673146:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.673147:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204920.673149:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204920.673164:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204920.673167:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204920.673168:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321279521 00000100:00000001:1.0:1788204920.673170:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321280776 00000100:00000001:1.0:1788204920.673171:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321279521 00000100:00000001:1.0:1788204920.673173:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321280776 00000100:00000001:1.0:1788204920.673173:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204920.673175:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.673176:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204920.673178:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204920/real 1788204920] req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204920.673183:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204920.673184:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204920.673185:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204920.673186:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204920.673187:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204920.673189:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204920.673190:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204920.673195:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673198:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204920.673199:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.673200:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.673201:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204920.673203:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673208:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204920.673211:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673217:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204920.673220:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.673221:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.673222:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.673224:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204920.673226:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204920.673228:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204920.673231:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204920.673233:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204920.673236:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204920.673243:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488768:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204920.673247:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204920.673251:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.673256:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204920.673257:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.673258:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204920.673260:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204920.673261:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204920.673263:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488896:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204920.673264:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204920.673266:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204920.673267:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204920.673268:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204920.673270:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204920.673271:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204920.673273:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204920.673275:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204920.673277:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204920.673280:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543488896, portal 4 00000100:00000001:1.0:1788204920.673282:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204920.673283:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:1.0:1788204920.674357:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674363:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204920.674364:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204920.674368:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204920.674371:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543488896, offset 0 00000400:00000200:1.0:1788204920.674374:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204920.674379:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204920.674382:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204920.674384:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204920.674386:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204920.674388:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204920.674389:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204920.674392:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471168:471168:256:4294967295] 192.168.201.118@tcp LPNI seq info [471027:471027:8:4294967295] 00000400:00000200:1.0:1788204920.674394:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204920.674398:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204920.674401:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204920.674404:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204920.674407:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204920.674409:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1788204920.674412:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204920.674414:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1788204920.674416:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204920.674418:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204920.674420:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204920.674423:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204920.674425:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204920.674427:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204920.674428:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204920.674430:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674439:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204920.674441:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204945 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674445:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.674446:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204920.674449:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204920.674451:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204920.674453:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204920.674455:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321279545 00000100:00000001:1.0:1788204920.674456:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321280776 00000100:00000001:1.0:1788204920.674457:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321279545 00000100:00000001:1.0:1788204920.674459:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321280776 00000100:00000001:1.0:1788204920.674460:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204920.674462:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.674463:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204920.674465:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204920/real 1788204920] req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204920.674469:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204920.674470:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204920.674472:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204920.674473:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204920.674474:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204920.674476:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204920.674477:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204920.674482:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674485:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204920.674486:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674488:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.674489:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204920.674491:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674495:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204920.674497:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674503:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204920.674506:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674507:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204920.674508:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674510:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204920.674511:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204920.674514:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204920.674517:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204920.674519:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204920.674523:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204920.674530:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543488896:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204920.674534:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674537:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204920.674540:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76680 x1875055543488512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674548:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204920.674550:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204920.674552:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204920.674555:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204920.674556:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204920.674558:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204920.674559:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204920.674561:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d000. 02000000:00000001:1.0:1788204920.674564:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204920.674566:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204920.674567:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76680. 00000100:00000001:1.0:1788204920.674569:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204920.674570:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674571:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204920.674574:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75c00 x1875055543488640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674578:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204920.674579:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204920.674579:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788204920.674581:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204920.674582:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204920.674582:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204920.674583:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204920.674584:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:1.0:1788204920.674586:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204920.674586:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204920.674587:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75c00. 00000100:00000001:1.0:1788204920.674588:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204920.674589:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674589:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204920.674591:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77480 x1875055543488768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674594:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204920.674595:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204920.674596:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204920.674597:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204920.674598:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204920.674598:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204920.674599:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204920.674600:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788204920.674602:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204920.674603:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204920.674603:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77480. 00000100:00000001:1.0:1788204920.674605:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204920.674606:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674607:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204920.674609:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543488896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204945 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204920.674612:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204920.674613:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204920.674613:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204920.674614:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204920.674615:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204920.674616:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204920.674616:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204920.674617:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:1.0:1788204920.674618:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204920.674619:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204920.674620:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788204920.674621:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204920.674621:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674623:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204920.674628:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204920.674628:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204920.674630:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204920.674631:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.500195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204921.500195:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204921.500202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204921.500203:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204921.500207:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204921.500208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.500210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204921.500214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204921.500215:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204921.500216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204921.500220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204921.500221:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204921.500222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204921.500226:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204921.500227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204921.500227:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204921.500232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.500233:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204921.500233:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204921.500237:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204921.500239:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204921.500240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204921.500242:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.500243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204921.500244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204921.500248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204921.500576:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204921.500579:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204921.500582:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204922.012210:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321280776, at 4321280832 00000100:00000010:2.0:1788204922.012227:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00000100:00000040:3.0:1788204922.012359:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321280776, at 4321280832 00000100:00000010:3.0:1788204922.012366:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647720. 00000100:00000001:1.0:1788204922.524175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204922.524178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204922.524182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204922.524183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204922.524184:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204922.524186:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788204922.524191:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204922.524192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204922.524192:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204922.524195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204922.524195:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204922.524196:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788204922.524199:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204922.524201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204922.524201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204922.524202:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204922.524204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204922.524206:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204922.524207:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204922.524208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204922.524210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204922.524210:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204922.524211:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204922.524214:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204922.524214:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204922.524217:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204922.524246:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204922.524249:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204922.524251:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204923.548208:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204923.548212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204923.548212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204923.548215:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204923.548216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204923.548218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204923.548219:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204923.548220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204923.548226:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204923.548227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204923.548227:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204923.548232:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204923.548233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204923.548233:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204923.548236:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788204923.548239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204923.548239:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204923.548240:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204923.548245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204923.548245:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204923.548246:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788204923.548248:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204923.548249:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204923.548250:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204923.548254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548255:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204923.548255:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548261:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204923.548268:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204923.548269:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204923.548272:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548274:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204923.548276:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548278:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204923.548285:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204923.548286:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204923.548287:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548289:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204923.548290:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548292:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204923.548294:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204923.548353:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204923.548356:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204923.548360:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204924.572189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204924.572189:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204924.572193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204924.572196:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204924.572197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204924.572199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204924.572199:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204924.572204:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204924.572205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204924.572207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204924.572209:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204924.572211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204924.572212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204924.572213:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204924.572219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204924.572219:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204924.572220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204924.572222:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204924.572224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204924.572224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204924.572226:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204924.572229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204924.572232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204924.572232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204924.572234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204924.572236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204924.572295:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204924.572299:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204924.572302:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204925.596202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204925.596212:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204925.596228:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204925.596233:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788204925.596245:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204925.596249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.596251:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204925.596263:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.596266:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.596358:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204925.596362:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204925.596363:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204925.596365:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204925.596367:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204925.596370:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204925.596372:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204925.596374:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204925.596376:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204925.597344:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204925.597348:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204925.597353:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204925.597354:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204925.597357:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204925.597358:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204925.597361:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204925.597362:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204925.597425:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204925.597437:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204925.597444:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204925.788181:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204925.788190:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204925.788192:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204925.788194:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204925.788198:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204925.788203:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26612 00000400:00000200:3.0:1788204925.788211:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204925.788217:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204925.788220:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204925.788222:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788224:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788226:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788228:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204925.788249:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204925.788251:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788204925.788253:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204925.788256:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788204925.788257:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788204925.788259:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788204925.788260:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788262:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788263:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788264:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204925.788266:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788204925.788268:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788269:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204925.788272:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204925.788275:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204925.788277:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788278:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204925.788279:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204925.788286:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788204925.788288:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788204925.788291:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204925.788292:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788300:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204925.788303:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204925.788313:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204925.788324:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:3.0:1788204925.788330:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204925.788338:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204925.788353:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204925.788357:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204925.788361:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204925.788380:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204925.788383:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204925.788386:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204925.788388:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204925.788389:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204925.788392:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204925.788394:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26612 00000400:00000200:3.0:1788204925.788399:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204925.788402:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204925.788405:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204925.788408:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788409:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788411:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788413:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204925.788416:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204925.788417:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:1.0:1788204925.788419:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788204925.788420:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204925.788422:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:1.0:1788204925.788426:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788204925.788426:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788204925.788431:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788204925.788431:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204925.788436:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788440:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204925.788442:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204925.788443:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788448:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204925.788454:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:1.0:1788204925.788460:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1788204925.788460:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204925.788465:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788465:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204925.788471:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204925.788472:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204925.788474:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204925.788476:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204925.788477:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788204925.788480:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543489024:192.168.201.118@tcp:250:kworker.0 02000000:00000001:3.0:1788204925.788481:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788485:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204925.788486:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1788204925.788488:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788204925.788490:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1788204925.788491:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.788493:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788494:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.788498:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1788204925.788500:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788204925.788502:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000010:1.0:1788204925.788504:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:3.0:1788204925.788505:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204925.788507:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.788508:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788509:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204925.788511:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204925.788513:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204925.788516:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:3.0:1788204925.788519:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204925.788523:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204925.788530:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204925.788530:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204925.788534:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204925.788535:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000100:00000040:3.0:1788204925.788540:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204925.788549:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204925.788551:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204925.788554:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788204925.788556:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204925.788557:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204925.788560:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204925.788562:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26612 00000400:00000200:3.0:1788204925.788567:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204925.788570:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204925.788572:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204925.788574:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788576:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788578:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788580:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204925.788583:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204925.788584:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788204925.788585:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204925.788587:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788204925.788589:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788204925.788591:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788204925.788594:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788596:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788598:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788600:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204925.788602:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788204925.788604:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788606:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204925.788608:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204925.788610:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204925.788611:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788613:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204925.788614:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204925.788616:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788617:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204925.788619:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788204925.788621:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204925.788622:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204925.788623:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788624:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204925.788625:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204925.788627:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204925.788629:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788204925.788631:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204925.788634:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204925.788638:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204925.788640:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204925.788642:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204925.788646:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204925.788648:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204925.788650:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204925.788651:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204925.788656:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204925.788658:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204925.788660:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26612 00000400:00000200:3.0:1788204925.788663:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204925.788664:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204925.788666:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204925.788667:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788669:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788670:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788671:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204925.788672:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204925.788673:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788204925.788674:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204925.788675:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788204925.788676:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788204925.788678:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788204925.788680:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788681:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788682:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788683:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204925.788685:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788204925.788686:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788687:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204925.788688:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204925.788689:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204925.788690:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788691:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204925.788692:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788204925.788694:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788695:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204925.788696:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788204925.788697:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204925.788702:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204925.788704:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788705:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204925.788706:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204925.788707:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204925.788711:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788204925.788713:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204925.788719:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204925.788726:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204925.788728:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204925.788731:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204925.788738:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204925.788741:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204925.788744:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204925.788745:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204925.788747:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204925.788749:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204925.788751:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26612 00000400:00000200:3.0:1788204925.788755:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204925.788757:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204925.788759:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204925.788762:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788763:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788765:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788766:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204925.788768:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204925.788769:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204925.788770:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204925.788771:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788204925.788772:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788204925.788773:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788204925.788775:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788776:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204925.788777:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788778:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204925.788780:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788204925.788781:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204925.788782:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204925.788783:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204925.788785:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204925.788786:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788787:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204925.788788:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788204925.788789:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788791:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204925.788794:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788204925.788795:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204925.788797:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204925.788797:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204925.788798:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204925.788799:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204925.788800:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204925.788803:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788204925.788805:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204925.788807:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204925.788812:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204925.788813:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204925.788815:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204925.788819:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204925.788822:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26622) 00000400:00000010:1.0:1788204925.789579:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204925.789586:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543489024, portal 25 00000100:00000001:1.0:1788204925.789588:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204925.789589:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788204925.789594:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.789599:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204925.789601:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204925.789603:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204925.789605:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543489024, offset 0 00000400:00000200:1.0:1788204925.789609:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204925.789627:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.789634:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204925.789640:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204925.789644:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204925.789647:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204925.789650:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204925.789654:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471169:471169:256:4294967295] 192.168.201.118@tcp LPNI seq info [471028:471028:8:4294967295] 00000400:00000200:1.0:1788204925.789657:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.789664:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204925.789669:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.789673:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204925.789678:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204925.789680:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204925.789684:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204925.789687:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788204925.789690:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.789691:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204925.789696:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204925.789699:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204925.789704:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204925.789706:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204925.789710:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204925.789714:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.789724:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.789728:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.789735:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.789737:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204925.789743:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204925.789747:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204925.789750:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321285894 00000100:00000010:1.0:1788204925.789755:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647720. 00000100:00000001:1.0:1788204925.789759:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788204925.789763:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321285894 00000100:00000010:1.0:1788204925.789765:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647600. 00000100:00000001:1.0:1788204925.789767:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788204925.789769:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204925.789771:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.789772:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204925.789775:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204925/real 1788204925] req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204925.789786:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204925.789787:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204925.789788:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204925.789790:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204925.789791:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204925.789792:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204925.789794:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204925.789800:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.789803:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204925.789804:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.789806:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.789807:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204925.789809:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.789813:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204925.789815:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.789819:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204925.789821:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.789822:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.789822:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.789824:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204925.789825:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204925.789827:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204925.789829:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204925.789831:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204925.789833:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204925.789838:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543489024:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788204925.789841:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.789842:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.789843:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543489024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.789847:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204925.789849:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204925.789851:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204925.789852:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204925.789853:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204925.789854:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204925.789855:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204925.789857:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788204925.789859:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204925.789861:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204925.789862:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788204925.789864:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204925.789865:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.789866:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.790857:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204925.790860:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.790863:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204925.790864:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204925.790866:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204925.790871:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.790879:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204925.790879:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.790881:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204925.790883:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204925.790885:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204925.790887:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489152:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204925.790889:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204925.790892:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204925.790893:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.790894:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204925.790897:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204925.790898:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204925.790901:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204925.790902:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204925.790905:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204925.790907:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543489152, portal 10 00000100:00000001:1.0:1788204925.790909:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204925.790910:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788204925.790914:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.790918:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204925.790919:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204925.790920:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204925.790922:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543489152, offset 0 00000400:00000200:1.0:1788204925.790924:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204925.790927:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.790929:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204925.790931:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204925.790933:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204925.790935:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204925.790936:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204925.790938:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471170:471170:256:4294967295] 192.168.201.118@tcp LPNI seq info [471029:471029:8:4294967295] 00000400:00000200:1.0:1788204925.790941:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.790945:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204925.790947:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.790949:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204925.790951:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204925.790954:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204925.790956:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204925.790958:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788204925.790960:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.790961:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204925.790963:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204925.790965:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204925.790967:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204925.790968:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204925.790970:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204925.790972:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.790977:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.790978:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.790982:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.790983:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204925.790985:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204925.790987:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788204925.790989:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204925.790991:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321284628 00000100:00000001:1.0:1788204925.790992:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321285894 00000100:00000001:1.0:1788204925.790994:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321284628 00000100:00000001:1.0:1788204925.790995:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321285894 00000100:00000001:1.0:1788204925.790996:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204925.790997:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.790999:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204925.791000:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204925/real 1788204925] req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204925.791005:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204925.791005:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204925.791007:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204925.791008:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204925.791009:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204925.791010:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204925.791012:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204925.791016:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791020:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204925.791020:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791021:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.791022:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204925.791025:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791030:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204925.791033:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791039:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204925.791041:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791043:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.791044:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791046:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204925.791048:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204925.791051:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204925.791054:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204925.791056:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204925.791060:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204925.791079:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489152:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204925.791083:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204925.791085:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791102:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204925.791103:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.791104:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204925.791105:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204925.791107:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204925.791109:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489280:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204925.791111:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204925.791112:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204925.791113:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.791114:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204925.791117:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204925.791118:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204925.791121:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204925.791122:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204925.791124:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204925.791126:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543489280, portal 10 00000100:00000001:1.0:1788204925.791127:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204925.791128:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788204925.791132:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791136:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204925.791137:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204925.791138:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204925.791140:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543489280, offset 0 00000400:00000200:1.0:1788204925.791142:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204925.791144:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.791146:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204925.791148:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204925.791150:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204925.791151:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204925.791153:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204925.791155:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471171:471171:256:4294967295] 192.168.201.118@tcp LPNI seq info [471030:471030:8:4294967295] 00000400:00000200:1.0:1788204925.791157:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.791161:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204925.791163:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.791165:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204925.791167:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204925.791169:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204925.791171:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204925.791173:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788204925.791176:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.791177:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204925.791179:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204925.791182:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204925.791184:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204925.791186:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204925.791188:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204925.791191:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791198:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.791201:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791207:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.791209:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204925.791212:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204925.791214:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204925.791218:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204925.791220:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321284630 00000100:00000001:1.0:1788204925.791222:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321285894 00000100:00000001:1.0:1788204925.791224:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321284630 00000100:00000001:1.0:1788204925.791227:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321285894 00000100:00000001:1.0:1788204925.791229:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204925.791230:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.791232:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204925.791236:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204925/real 1788204925] req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204925.791244:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204925.791245:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204925.791246:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204925.791249:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204925.791250:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204925.791252:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204925.791255:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204925.791264:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791270:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204925.791271:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791273:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.791274:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204925.791278:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791285:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204925.791288:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791292:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204925.791294:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791295:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.791295:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791297:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204925.791299:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204925.791300:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204925.791302:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204925.791304:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204925.791306:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204925.791311:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489280:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204925.791313:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204925.791315:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791318:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204925.791319:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.791320:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204925.791321:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204925.791323:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204925.791324:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489408:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204925.791326:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204925.791327:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204925.791328:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.791329:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204925.791331:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788204925.791333:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204925.791335:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204925.791336:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204925.791338:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204925.791340:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543489408, portal 4 00000100:00000001:1.0:1788204925.791341:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204925.791342:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788204925.791345:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791349:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204925.791351:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204925.791352:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204925.791353:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543489408, offset 0 00000400:00000200:1.0:1788204925.791355:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204925.791358:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.791360:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204925.791361:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204925.791363:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204925.791364:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204925.791365:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204925.791367:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471172:471172:256:4294967295] 192.168.201.118@tcp LPNI seq info [471031:471031:8:4294967295] 00000400:00000200:1.0:1788204925.791370:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.791373:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204925.791376:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.791377:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204925.791380:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204925.791381:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204925.791384:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204925.791385:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788204925.791388:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.791389:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204925.791391:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204925.791393:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204925.791394:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204925.791395:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204925.791397:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204925.791399:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791404:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.791407:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791418:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.791420:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204925.791423:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204925.791426:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204925.791429:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204925.791431:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321284639 00000100:00000001:1.0:1788204925.791433:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321285894 00000100:00000001:1.0:1788204925.791435:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321284639 00000100:00000001:1.0:1788204925.791437:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321285894 00000100:00000001:1.0:1788204925.791439:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204925.791441:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.791443:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204925.791445:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204925/real 1788204925] req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204925.791450:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204925.791451:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204925.791452:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204925.791454:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204925.791455:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204925.791456:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204925.791458:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204925.791463:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791466:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204925.791467:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791468:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.791468:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204925.791470:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791474:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204925.791477:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791480:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204925.791482:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791482:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.791483:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791484:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204925.791485:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204925.791486:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204925.791488:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204925.791489:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204925.791491:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204925.791495:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489408:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204925.791497:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204925.791499:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791503:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204925.791504:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.791505:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204925.791506:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204925.791507:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204925.791508:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489536:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204925.791510:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204925.791511:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204925.791513:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204925.791514:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204925.791516:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204925.791517:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204925.791519:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204925.791521:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204925.791522:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204925.791524:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543489536, portal 4 00000100:00000001:1.0:1788204925.791525:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204925.791526:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788204925.791529:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791533:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204925.791534:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204925.791535:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204925.791537:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543489536, offset 0 00000400:00000200:1.0:1788204925.791539:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204925.791541:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.791543:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204925.791544:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204925.791546:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204925.791547:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204925.791548:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204925.791550:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471173:471173:256:4294967295] 192.168.201.118@tcp LPNI seq info [471032:471032:8:4294967295] 00000400:00000200:1.0:1788204925.791553:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204925.791556:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204925.791559:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.791561:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204925.791563:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204925.791564:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1788204925.791567:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204925.791568:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1788204925.791571:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204925.791572:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204925.791574:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204925.791576:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204925.791578:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204925.791578:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204925.791580:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204925.791581:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791586:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.791587:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204950 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791591:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.791592:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204925.791594:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204925.791595:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204925.791597:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204925.791598:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321284662 00000100:00000001:1.0:1788204925.791600:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321285894 00000100:00000001:1.0:1788204925.791601:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321284662 00000100:00000001:1.0:1788204925.791602:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321285894 00000100:00000001:1.0:1788204925.791604:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204925.791605:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.791606:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204925.791607:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204925/real 1788204925] req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204925.791611:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204925.791612:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204925.791613:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204925.791614:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204925.791615:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204925.791616:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204925.791618:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204925.791623:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791626:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204925.791627:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791627:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.791628:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204925.791630:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791633:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204925.791635:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791639:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204925.791640:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791640:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204925.791641:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791642:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204925.791643:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204925.791644:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204925.791646:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204925.791647:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204925.791649:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204925.791654:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489536:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204925.791656:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791657:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.791659:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543489152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791663:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204925.791664:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204925.791665:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204925.791666:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204925.791667:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204925.791668:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204925.791669:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204925.791670:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788204925.791672:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204925.791673:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204925.791674:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788204925.791675:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204925.791676:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791677:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.791680:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543489280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791686:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204925.791687:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204925.791688:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204925.791690:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204925.791691:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204925.791692:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204925.791693:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204925.791695:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788204925.791696:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204925.791697:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204925.791698:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788204925.791701:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204925.791702:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791704:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.791707:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543489408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791712:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204925.791714:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204925.791715:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788204925.791717:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204925.791718:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204925.791720:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204925.791722:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204925.791723:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788204925.791725:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204925.791726:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204925.791727:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788204925.791729:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204925.791730:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791731:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204925.791734:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543489536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204950 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204925.791740:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204925.791742:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204925.791743:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204925.791745:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204925.791746:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204925.791746:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204925.791748:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204925.791748:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788204925.791750:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204925.791750:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204925.791751:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788204925.791752:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204925.791753:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791754:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204925.791756:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204925.791757:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204925.791758:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204925.791759:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204926.620198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204926.620200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204926.620207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204926.620208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204926.620212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204926.620215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204926.620218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204926.620218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204926.620222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204926.620224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204926.620225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204926.620229:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204926.620231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204926.620235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204926.620236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204926.620240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204926.620243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204926.620293:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204926.620297:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204926.620300:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204926.620335:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204926.620338:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204926.620340:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204926.620343:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204926.620345:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204926.620347:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204926.620349:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204926.620351:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204926.620353:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204927.132178:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321285894, at 4321285952 00000100:00000010:2.0:1788204927.132193:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647720. 00000100:00000040:3.0:1788204927.132215:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321285894, at 4321285952 00000100:00000010:3.0:1788204927.132229:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00000100:00000001:1.0:1788204927.644219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204927.644219:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204927.644227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204927.644230:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204927.644230:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204927.644233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204927.644234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204927.644235:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204927.644242:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204927.644243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204927.644244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204927.644249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204927.644249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204927.644250:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204927.644253:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204927.644256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204927.644256:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204927.644257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204927.644261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204927.644262:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204927.644263:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204927.644268:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204927.644269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204927.644271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204927.644275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204927.644275:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204927.644342:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204927.644345:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204927.644349:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204928.668154:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204928.668162:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204928.668164:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204928.668167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204928.668169:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204928.668174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204928.668175:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204928.668179:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204928.668180:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204928.668185:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204928.668186:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204928.668189:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204928.668191:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204928.668193:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204928.668195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204928.668199:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204928.668202:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204928.668203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204928.668205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204928.668212:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204928.668216:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204928.668220:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204928.668225:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204928.668228:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204928.668230:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204928.668232:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204928.668237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204928.668239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204928.668243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204928.668244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204928.668246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204928.668247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204928.668283:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204928.668285:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204928.668287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204929.692179:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204929.692179:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204929.692188:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204929.692189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204929.692190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204929.692193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.692193:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204929.692197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204929.692201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204929.692206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204929.692206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788204929.692207:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204929.692210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204929.692211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204929.692212:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204929.692216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204929.692217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204929.692217:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204929.692219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204929.692221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.692224:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204929.692225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204929.692227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204929.692228:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204929.692231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204929.692233:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204929.692289:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204929.692292:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204929.692294:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204930.716235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204930.716235:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204930.716241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204930.716246:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204930.716247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.716247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204930.716250:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.716250:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204930.716258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204930.716258:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204930.716259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204930.716262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.716264:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204930.716265:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204930.716267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204930.716270:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.716270:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204930.716271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.716277:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204930.716279:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204930.716280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204930.716283:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.716283:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204930.716284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.716289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.716291:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204930.716378:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204930.716382:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204930.716386:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204930.908262:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204930.908276:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204930.908280:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204930.908285:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204930.908290:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204930.908299:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26617 00000400:00000200:1.0:1788204930.908322:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204930.908331:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204930.908335:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204930.908338:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908341:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.908344:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908347:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204930.908351:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204930.908354:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788204930.908357:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204930.908361:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788204930.908364:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788204930.908367:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788204930.908370:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908372:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.908374:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908376:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204930.908380:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788204930.908383:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908392:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204930.908396:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204930.908400:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204930.908404:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908407:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204930.908410:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204930.908423:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788204930.908427:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788204930.908431:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204930.908433:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908437:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204930.908440:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204930.908445:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204930.908454:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8400. 00000100:00000001:1.0:1788204930.908461:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204930.908471:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204930.908482:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204930.908488:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204930.908493:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204930.908519:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204930.908523:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204930.908527:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204930.908529:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204930.908531:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204930.908540:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204930.908544:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26617 00000400:00000200:1.0:1788204930.908549:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204930.908553:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204930.908553:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1788204930.908558:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204930.908560:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788204930.908564:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204930.908565:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788204930.908568:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.908571:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908574:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204930.908578:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204930.908580:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788204930.908583:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:3.0:1788204930.908584:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788204930.908586:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788204930.908591:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788204930.908596:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788204930.908602:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204930.908604:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204930.908611:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:3.0:1788204930.908611:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788204930.908616:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788204930.908616:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788204930.908621:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:3.0:1788204930.908623:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908626:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204930.908630:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00100000:3.0:1788204930.908631:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543489664:192.168.201.118@tcp:250:kworker.0 00000100:00080000:1.0:1788204930.908636:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204930.908640:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788204930.908641:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204930.908644:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204930.908647:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204930.908651:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204930.908652:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204930.908654:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788204930.908656:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908659:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000010:3.0:1788204930.908662:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 00000008:00100000:1.0:1788204930.908664:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204930.908665:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788204930.908671:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908674:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204930.908676:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:1.0:1788204930.908680:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000400:00000010:3.0:1788204930.908682:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000020:00000040:1.0:1788204930.908685:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788204930.908689:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204930.908690:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908692:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1788204930.908693:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 02000000:00000001:1.0:1788204930.908695:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204930.908698:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204930.908703:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000200:3.0:1788204930.908703:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543489664, portal 25 00000100:00000001:1.0:1788204930.908709:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204930.908716:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204930.908716:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204930.908720:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788204930.908734:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.908748:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788204930.908750:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788204930.908752:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908756:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204930.908758:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204930.908763:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543489664, offset 0 00000100:00000040:1.0:1788204930.908764:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204930.908774:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204930.908778:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204930.908781:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788204930.908783:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204930.908784:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204930.908787:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204930.908790:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26617 00000400:00000200:1.0:1788204930.908802:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204930.908805:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204930.908807:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204930.908810:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908812:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.908814:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908816:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204930.908818:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204930.908819:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788204930.908821:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204930.908823:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788204930.908825:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788204930.908827:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788204930.908831:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908832:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.908834:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908836:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204930.908839:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788204930.908841:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908843:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204930.908845:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204930.908848:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204930.908850:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908852:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204930.908854:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204930.908864:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908868:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204930.908875:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000010:1.0:1788204930.908878:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788204930.908884:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788204930.908888:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1788204930.908888:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:1.0:1788204930.908892:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.908895:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204930.908898:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204930.908899:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 02000000:00000001:1.0:1788204930.908904:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204930.908906:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204930.908914:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000010:1.0:1788204930.908916:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9400. 00000400:00000200:3.0:1788204930.908918:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204930.908925:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908926:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204930.908933:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471174:471174:256:4294967295] 192.168.201.118@tcp LPNI seq info [471033:471033:8:4294967295] 00000100:00100000:1.0:1788204930.908940:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204930.908946:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.908968:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:1.0:1788204930.908971:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:3.0:1788204930.908987:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788204930.908989:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204930.908998:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1788204930.909000:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000100:00000001:1.0:1788204930.909011:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204930.909012:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:1.0:1788204930.909015:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204930.909021:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204930.909023:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.0:1788204930.909023:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00080000:1.0:1788204930.909027:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204930.909035:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.1:1788204930.909039:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:1.0:1788204930.909040:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26617 00000800:00000200:3.1:1788204930.909050:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000400:00000200:1.0:1788204930.909058:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000100:3.0:1788204930.909063:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.909065:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204930.909070:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204930.909077:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204930.909082:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:3.0:1788204930.909083:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204930.909121:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204930.909126:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204930.909132:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204930.909144:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204930.909147:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204930.909153:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:3.0:1788204930.909155:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204930.909159:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.909167:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.909173:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204930.909178:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204930.909179:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204930.909186:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000100:00000001:3.0:1788204930.909186:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1788204930.909191:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204930.909194:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788204930.909196:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:3.0:1788204930.909199:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788204930.909200:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788204930.909210:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:1.0:1788204930.909211:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204930.909217:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321291014 00000100:00000001:1.0:1788204930.909218:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.909225:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000010:3.0:1788204930.909227:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc540. 00000100:00000001:1.0:1788204930.909230:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204930.909235:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000020:00000040:1.0:1788204930.909237:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000001:3.0:1788204930.909243:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321291014 00000100:00080000:1.0:1788204930.909244:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000010:3.0:1788204930.909248:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc900. 00000100:00000001:1.0:1788204930.909252:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909252:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:1.0:1788204930.909259:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204930.909260:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204930.909266:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204930.909268:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204930.909269:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788204930.909274:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204930.909276:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204930/real 1788204930] req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204930.909281:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204930.909289:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000400:00000001:3.0:1788204930.909290:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:1.0:1788204930.909295:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.909298:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788204930.909298:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204930.909305:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00000010:1.0:1788204930.909307:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000400:00000200:3.0:1788204930.909309:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000020:00000040:1.0:1788204930.909312:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000400:00000010:3.0:1788204930.909313:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204930.909321:0:256132:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1788204930.909323:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204930.909328:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204930.909328:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204930.909339:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204930.909345:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:3.0:1788204930.909346:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204930.909351:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909353:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204930.909357:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.909359:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:1.0:1788204930.909360:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000001:3.0:1788204930.909361:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204930.909369:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204930.909370:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909382:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204930.909384:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204930.909390:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204930.909429:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204930.909430:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204930.909436:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909437:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.909442:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.909446:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204930.909447:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909453:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204930.909457:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204930.909464:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204930.909466:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204930.909469:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204930.909473:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204930.909474:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204930.909479:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204930.909480:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204930.909482:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26617 00000400:00000200:1.0:1788204930.909491:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788204930.909492:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204930.909497:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:1.0:1788204930.909498:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204930.909504:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204930.909509:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:3.0:1788204930.909509:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204930.909518:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.909527:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.909537:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204930.909543:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00100000:3.0:1788204930.909544:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543489664:192.168.201.118@tcp:250:kworker.0 00000020:00000040:1.0:1788204930.909548:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788204930.909554:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788204930.909555:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1788204930.909562:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:3.0:1788204930.909563:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1788204930.909568:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788204930.909573:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788204930.909575:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543489664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204930.909580:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204930.909587:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204930.909592:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204930.909597:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1788204930.909598:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:3.0:1788204930.909603:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:1.0:1788204930.909604:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 02000000:00000010:3.0:1788204930.909609:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 00000100:00080000:1.0:1788204930.909610:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788204930.909615:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1788204930.909616:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204930.909619:0:256132:0:(genops.c:1239:class_import_put()) Process entered 02000000:00000001:1.0:1788204930.909625:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000040:3.0:1788204930.909625:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204930.909631:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204930.909634:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788204930.909635:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204930.909637:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204930.909640:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.909645:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204930.909646:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000008:00100000:1.0:1788204930.909648:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000010:3.0:1788204930.909648:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000008:00000001:1.0:1788204930.909657:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.909659:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204930.909662:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788204930.909665:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788204930.909668:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204930.909669:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204930.909671:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204930.909672:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204930.909675:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909676:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204930.909680:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.909683:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1788204930.909684:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8800. 00000100:00000001:1.0:1788204930.909690:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909694:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1788204930.909698:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204930.909698:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909708:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204930.909712:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204930.909717:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204930.909717:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788204930.909723:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204930.909725:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909738:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1788204930.909739:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909740:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204930.909748:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204930.909752:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204930.909760:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204930.909771:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489792:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204930.909775:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909776:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1788204930.909787:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26627) 02000000:00000001:3.0:1788204930.909787:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204930.909792:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204930.909795:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204930.909799:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204930.909800:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204930.909804:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204930.909806:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204930.909812:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204930.909815:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543489792, portal 10 00000100:00000001:3.0:1788204930.909817:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204930.909818:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788204930.909823:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909828:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204930.909829:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204930.909831:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204930.909832:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543489792, offset 0 00000400:00000200:3.0:1788204930.909835:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204930.909837:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.909840:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204930.909842:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204930.909844:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204930.909845:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204930.909847:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204930.909849:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471175:471175:256:4294967295] 192.168.201.118@tcp LPNI seq info [471034:471034:8:4294967295] 00000400:00000200:3.0:1788204930.909851:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.909855:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204930.909858:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.909860:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204930.909863:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204930.909865:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204930.909868:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204930.909870:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788204930.909872:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.909873:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204930.909875:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204930.909878:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204930.909879:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204930.909880:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204930.909882:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204930.909884:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909889:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204930.909890:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909894:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909895:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204930.909897:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204930.909900:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204930.909901:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204930.909903:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321289747 00000100:00000001:3.0:1788204930.909904:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321291014 00000100:00000001:3.0:1788204930.909906:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321289747 00000100:00000001:3.0:1788204930.909907:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321291014 00000100:00000001:3.0:1788204930.909908:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204930.909909:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.909910:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204930.909912:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204930/real 1788204930] req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204930.909917:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204930.909918:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204930.909919:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204930.909920:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204930.909921:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204930.909922:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204930.909924:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204930.909928:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909932:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204930.909933:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.909934:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.909935:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204930.909937:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909941:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204930.909944:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.909951:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204930.909953:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.909954:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.909956:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.909957:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204930.909959:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204930.909962:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204930.909965:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204930.909967:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204930.909972:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204930.909990:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489792:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204930.909994:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204930.909998:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910004:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204930.910006:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204930.910008:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204930.910010:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204930.910011:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204930.910014:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489920:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204930.910017:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204930.910020:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204930.910021:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204930.910023:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204930.910026:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204930.910028:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204930.910031:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204930.910034:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204930.910037:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204930.910040:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543489920, portal 10 00000100:00000001:3.0:1788204930.910042:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204930.910042:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788204930.910047:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910051:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204930.910052:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204930.910054:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204930.910057:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543489920, offset 0 00000400:00000200:3.0:1788204930.910061:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204930.910064:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.910067:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204930.910070:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204930.910073:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204930.910075:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204930.910077:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204930.910080:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471176:471176:256:4294967295] 192.168.201.118@tcp LPNI seq info [471035:471035:8:4294967295] 00000400:00000200:3.0:1788204930.910084:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.910113:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204930.910119:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.910123:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204930.910126:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204930.910129:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204930.910133:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204930.910135:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788204930.910139:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.910140:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204930.910142:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204930.910144:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204930.910146:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204930.910147:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204930.910149:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204930.910151:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910156:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204930.910158:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910162:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.910163:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204930.910165:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204930.910167:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204930.910169:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204930.910170:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321289749 00000100:00000001:3.0:1788204930.910172:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321291014 00000100:00000001:3.0:1788204930.910173:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321289749 00000100:00000001:3.0:1788204930.910175:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321291014 00000100:00000001:3.0:1788204930.910176:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204930.910177:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.910178:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204930.910180:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204930/real 1788204930] req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204930.910184:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204930.910185:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204930.910186:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204930.910187:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204930.910188:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204930.910190:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204930.910192:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204930.910196:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910199:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204930.910200:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910201:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.910202:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204930.910204:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910207:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204930.910209:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910213:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204930.910214:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910215:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.910216:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910217:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204930.910219:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204930.910221:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204930.910222:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204930.910224:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204930.910226:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204930.910230:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543489920:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204930.910232:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204930.910234:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910238:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204930.910239:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204930.910241:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204930.910242:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204930.910243:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204930.910244:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490048:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204930.910246:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204930.910247:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204930.910248:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204930.910250:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204930.910252:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204930.910253:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204930.910254:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204930.910256:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204930.910258:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204930.910259:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543490048, portal 4 00000100:00000001:3.0:1788204930.910261:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204930.910262:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788204930.910265:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910269:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204930.910270:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204930.910272:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204930.910273:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543490048, offset 0 00000400:00000200:3.0:1788204930.910275:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204930.910277:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.910279:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204930.910280:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204930.910282:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204930.910284:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204930.910285:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204930.910287:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471177:471177:256:4294967295] 192.168.201.118@tcp LPNI seq info [471036:471036:8:4294967295] 00000400:00000200:3.0:1788204930.910289:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.910293:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204930.910295:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.910297:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204930.910299:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204930.910301:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204930.910303:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204930.910304:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788204930.910306:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.910308:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204930.910309:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204930.910311:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204930.910313:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204930.910314:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204930.910315:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204930.910317:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910325:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204930.910326:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910330:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.910331:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204930.910332:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204930.910334:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204930.910335:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204930.910336:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321289758 00000100:00000001:3.0:1788204930.910338:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321291014 00000100:00000001:3.0:1788204930.910339:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321289758 00000100:00000001:3.0:1788204930.910340:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321291014 00000100:00000001:3.0:1788204930.910342:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204930.910343:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.910344:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204930.910346:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204930/real 1788204930] req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204930.910349:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204930.910350:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204930.910351:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204930.910352:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204930.910353:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204930.910354:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204930.910356:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204930.910361:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910364:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204930.910365:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910366:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.910367:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204930.910369:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910372:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204930.910374:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910378:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204930.910379:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910380:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.910380:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910381:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204930.910382:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204930.910383:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204930.910385:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204930.910386:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204930.910388:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204930.910392:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490048:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204930.910394:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910395:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204930.910397:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543489792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910401:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204930.910402:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204930.910403:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204930.910404:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204930.910405:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204930.910406:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204930.910407:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204930.910408:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:3.0:1788204930.910410:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204930.910411:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204930.910412:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788204930.910414:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204930.910414:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910415:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204930.910417:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543489920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910421:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204930.910421:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204930.910422:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204930.910423:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204930.910424:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204930.910425:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204930.910426:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204930.910426:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:3.0:1788204930.910427:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204930.910428:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204930.910429:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788204930.910430:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204930.910430:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910431:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204930.910433:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543490048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910437:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204930.910438:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204930.910439:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204930.910441:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204930.910442:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204930.910443:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204930.910444:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204930.910446:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:3.0:1788204930.910447:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204930.910449:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204930.910450:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788204930.910454:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204930.910455:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910457:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910461:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204930.910462:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.910464:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204930.910465:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204930.910466:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204930.910471:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910478:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204930.910479:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204930.910481:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204930.910483:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204930.910485:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204930.910487:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490176:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204930.910490:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204930.910492:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204930.910494:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204930.910496:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204930.910499:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204930.910500:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204930.910503:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204930.910504:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204930.910506:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204930.910508:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543490176, portal 4 00000100:00000001:3.0:1788204930.910509:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204930.910510:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788204930.910514:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910518:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204930.910519:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204930.910520:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204930.910522:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543490176, offset 0 00000400:00000200:3.0:1788204930.910524:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204930.910526:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.910528:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204930.910530:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204930.910532:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204930.910533:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204930.910535:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204930.910537:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471178:471178:256:4294967295] 192.168.201.118@tcp LPNI seq info [471037:471037:8:4294967295] 00000400:00000200:3.0:1788204930.910539:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204930.910542:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204930.910545:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.910547:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204930.910549:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204930.910551:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788204930.910554:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204930.910555:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788204930.910557:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204930.910558:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204930.910561:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204930.910564:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204930.910567:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204930.910568:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204930.910570:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204930.910573:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910580:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204930.910583:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204955 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910589:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.910591:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204930.910594:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204930.910597:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204930.910600:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204930.910602:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321289781 00000100:00000001:3.0:1788204930.910604:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321291014 00000100:00000001:3.0:1788204930.910606:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321289781 00000100:00000001:3.0:1788204930.910608:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321291014 00000100:00000001:3.0:1788204930.910610:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204930.910611:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.910613:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204930.910615:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204930/real 1788204930] req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204930.910621:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204930.910621:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204930.910622:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204930.910624:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204930.910625:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204930.910626:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204930.910628:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204930.910633:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910636:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204930.910637:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910638:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.910638:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204930.910640:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910643:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204930.910645:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910649:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204930.910651:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910652:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204930.910652:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910653:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204930.910655:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204930.910656:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204930.910658:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204930.910659:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204930.910661:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204930.910665:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490176:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204930.910667:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910668:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204930.910669:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543490176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204955 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204930.910673:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204930.910674:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204930.910675:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204930.910676:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204930.910677:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204930.910678:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204930.910679:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204930.910680:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:3.0:1788204930.910681:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204930.910682:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204930.910683:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788204930.910686:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204930.910687:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910688:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204930.910690:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204930.910691:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204930.910692:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204930.910693:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204931.740229:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204931.740229:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204931.740234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204931.740241:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204931.740242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204931.740242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204931.740246:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204931.740246:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204931.740251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204931.740254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204931.740254:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204931.740255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204931.740261:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204931.740263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204931.740267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204931.740268:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204931.740268:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204931.740273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204931.740274:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204931.740275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204931.740278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204931.740281:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204931.740281:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204931.740285:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204931.740287:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204931.740291:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204931.740363:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204931.740368:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204931.740372:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204932.252224:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321291014, at 4321291072 00000100:00000040:3.0:1788204932.252224:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321291014, at 4321291072 00000100:00000010:3.0:1788204932.252245:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc900. 00000100:00000010:2.0:1788204932.252252:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc540. 00000100:00000001:1.0:1788204932.764221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204932.764221:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204932.764229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204932.764231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204932.764231:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204932.764234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204932.764236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204932.764236:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204932.764242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204932.764243:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204932.764246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204932.764248:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204932.764250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204932.764250:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204932.764252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204932.764254:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204932.764256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204932.764259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204932.764263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204932.764266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204932.764268:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204932.764270:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204932.764273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204932.764273:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204932.764276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204932.764278:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204932.764339:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204932.764342:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204932.764345:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204933.788175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204933.788175:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204933.788186:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204933.788187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204933.788191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204933.788191:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204933.788200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204933.788200:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204933.788207:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204933.788208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204933.788212:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204933.788213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204933.788217:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204933.788218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204933.788222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204933.788222:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204933.788230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204933.788230:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204933.788284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204933.788293:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204933.788296:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204933.788299:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788303:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204933.788306:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788308:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788310:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788314:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204933.788316:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204933.788317:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204933.788318:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788320:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204933.788321:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788323:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788325:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788329:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204933.788331:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204933.788332:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204933.788334:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788335:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204933.788336:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788338:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204933.788340:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204933.788393:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204933.788396:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204933.788398:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204934.812232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204934.812233:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204934.812239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204934.812242:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204934.812244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204934.812247:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204934.812248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204934.812249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204934.812258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204934.812260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204934.812260:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204934.812264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204934.812268:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204934.812268:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204934.812270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204934.812276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204934.812279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204934.812279:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204934.812283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204934.812287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204934.812289:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204934.812290:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204934.812297:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204934.812299:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204934.812302:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204934.812303:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204934.812383:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204934.812388:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204934.812392:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204935.836209:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204935.836210:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204935.836219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204935.836220:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204935.836221:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204935.836225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204935.836225:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204935.836227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204935.836234:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204935.836234:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204935.836238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204935.836241:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204935.836241:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204935.836243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204935.836247:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204935.836247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204935.836250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204935.836253:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204935.836253:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204935.836254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204935.836257:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204935.836259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204935.836261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204935.836264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204935.836264:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204935.836265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204935.836325:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204935.836328:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204935.836331:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204936.028211:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204936.028221:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204936.028224:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204936.028228:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204936.028232:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204936.028240:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26622 00000400:00000200:2.0:1788204936.028250:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204936.028257:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204936.028260:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204936.028263:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028265:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204936.028267:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028268:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204936.028272:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204936.028273:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204936.028276:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204936.028279:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204936.028280:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204936.028282:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204936.028284:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028286:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204936.028287:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028288:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204936.028290:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204936.028292:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028294:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204936.028296:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204936.028299:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204936.028301:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028303:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204936.028305:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204936.028315:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788204936.028317:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204936.028321:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204936.028322:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028326:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204936.028328:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204936.028333:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204936.028340:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204936.028347:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204936.028355:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204936.028364:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204936.028367:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204936.028371:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204936.028388:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204936.028391:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204936.028394:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204936.028395:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204936.028396:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204936.028397:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204936.028400:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26622 00000400:00000200:2.0:1788204936.028402:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204936.028404:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204936.028406:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204936.028408:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028410:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204936.028411:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028413:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204936.028414:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:3.0:1788204936.028414:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1788204936.028416:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204936.028419:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788204936.028420:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1788204936.028421:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204936.028424:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788204936.028425:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788204936.028426:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204936.028430:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028432:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204936.028435:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:3.0:1788204936.028435:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204936.028439:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204936.028442:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788204936.028447:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028451:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204936.028454:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204936.028455:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204936.028458:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204936.028460:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204936.028461:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204936.028465:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.028465:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204936.028467:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204936.028470:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788204936.028472:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788204936.028475:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543490304:192.168.201.118@tcp:250:kworker.0 00000008:00000001:2.0:1788204936.028476:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028480:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.028482:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1788204936.028483:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788204936.028486:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204936.028487:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204936.028487:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204936.028490:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.028491:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028492:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204936.028494:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788204936.028495:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204936.028498:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204936.028501:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000010:2.0:1788204936.028502:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788204936.028506:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028508:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204936.028515:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204936.028515:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204936.028521:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:2.0:1788204936.028524:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204936.028526:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204936.028529:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788204936.028531:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000001:2.0:1788204936.028537:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204936.028540:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204936.028542:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204936.028543:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204936.028544:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1788204936.028544:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543490304, portal 25 00000100:00000001:2.0:1788204936.028547:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204936.028551:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26622 00000400:00000200:2.0:1788204936.028554:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204936.028556:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204936.028557:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204936.028558:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028560:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204936.028560:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1788204936.028563:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204936.028565:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000100:00000001:2.0:1788204936.028566:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204936.028570:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204936.028572:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204936.028574:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204936.028575:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204936.028577:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204936.028579:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788204936.028580:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204936.028583:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028586:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204936.028589:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028593:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204936.028596:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204936.028600:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028603:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.028603:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788204936.028606:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204936.028608:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1788204936.028611:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000001:2.0:1788204936.028612:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028615:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204936.028617:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204936.028619:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204936.028619:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000001:2.0:1788204936.028620:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204936.028626:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000100:00000200:3.0:1788204936.028626:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543490304, offset 0 00000020:00000040:2.0:1788204936.028630:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204936.028633:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204936.028633:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028634:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204936.028635:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204936.028637:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204936.028641:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 02000000:00000010:2.0:1788204936.028642:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788204936.028646:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204936.028649:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204936.028654:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204936.028656:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204936.028659:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204936.028662:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204936.028668:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:2.0:1788204936.028669:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204936.028674:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204936.028678:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788204936.028678:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788204936.028680:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204936.028683:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204936.028685:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204936.028687:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00080000:2.0:1788204936.028689:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26622 00000400:00000200:3.0:1788204936.028695:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204936.028700:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204936.028706:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471179:471179:256:4294967295] 192.168.201.118@tcp LPNI seq info [471038:471038:8:4294967295] 00000400:00000200:3.0:1788204936.028711:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.028722:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000400:00000200:2.0:1788204936.028731:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.0:1788204936.028732:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000400:00000200:2.0:1788204936.028738:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000010:3.0:1788204936.028740:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000100:00080000:2.0:1788204936.028741:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204936.028747:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028755:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788204936.028756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788204936.028759:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028763:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.0:1788204936.028763:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1788204936.028767:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204936.028770:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204936.028774:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:3.1:1788204936.028775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:2.0:1788204936.028777:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204936.028782:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:3.1:1788204936.028782:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000020:00000001:2.0:1788204936.028786:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000100:3.0:1788204936.028792:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000040:2.0:1788204936.028793:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028797:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:3.0:1788204936.028798:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000100:00000040:2.0:1788204936.028800:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028804:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204936.028807:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:3.0:1788204936.028809:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1788204936.028810:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204936.028816:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204936.028820:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788204936.028820:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1788204936.028824:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204936.028828:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204936.028829:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000100:00000001:2.0:1788204936.028830:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204936.028835:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000400:00000010:3.0:1788204936.028835:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000008:00000001:2.0:1788204936.028841:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028843:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.028844:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:2.0:1788204936.028848:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204936.028853:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000200:3.0:1788204936.028853:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204936.028857:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204936.028860:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028864:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204936.028866:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204936.028869:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204936.028875:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204936.028881:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.028882:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1788204936.028898:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204936.028902:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204936.028918:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204936.028922:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028923:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.028928:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1788204936.028932:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.028937:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.028945:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204936.028947:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:2.0:1788204936.028949:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204936.028952:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321296133 00000100:00080000:2.0:1788204936.028954:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204936.028958:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204936.028960:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:3.0:1788204936.028962:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc900. 00000100:00000001:2.0:1788204936.028963:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204936.028968:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26622 00000400:00000200:2.0:1788204936.028973:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204936.028974:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000400:00000200:2.0:1788204936.028978:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204936.028981:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204936.028986:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204936.028987:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321296133 00000100:00000001:2.0:1788204936.028988:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204936.028993:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000010:3.0:1788204936.028994:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc240. 00000100:00000001:2.0:1788204936.028997:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204936.029000:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000020:00000001:2.0:1788204936.029002:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204936.029005:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000001:3.0:1788204936.029007:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788204936.029008:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204936.029011:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:3.0:1788204936.029013:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204936.029014:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204936.029019:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788204936.029021:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1788204936.029023:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204936.029026:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204936.029030:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00100000:3.0:1788204936.029030:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204936/real 1788204936] req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204936.029033:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204936.029036:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204936.029039:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204936.029043:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204936.029047:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204936.029050:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204936.029052:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.029054:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000001:3.0:1788204936.029054:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00100000:2.0:1788204936.029055:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000400:00000001:3.0:1788204936.029058:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:2.0:1788204936.029059:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.029062:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204936.029063:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00000010:2.0:1788204936.029065:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204936.029070:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000400:00000200:3.0:1788204936.029070:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 02000000:00000001:2.0:1788204936.029074:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000010:3.0:1788204936.029075:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 02000000:00000001:2.0:1788204936.029076:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204936.029080:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204936.029082:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204936.029082:0:256132:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1788204936.029085:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204936.029120:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788204936.029124:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204936.029125:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204936.029129:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204936.029153:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204936.029155:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204936.029159:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204936.029159:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204936.029178:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.029184:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1788204936.029185:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26632) 00000100:00000001:3.0:1788204936.029190:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029194:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.029197:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204936.029203:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029211:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204936.029217:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029225:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204936.029228:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029229:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.029230:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029233:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204936.029236:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204936.029239:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204936.029243:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204936.029246:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204936.029250:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204936.029260:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543490304:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788204936.029265:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029267:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.029270:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543490304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029277:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204936.029280:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204936.029283:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204936.029286:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204936.029288:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204936.029290:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204936.029293:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204936.029296:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067ec00. 02000000:00000001:3.0:1788204936.029301:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204936.029303:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204936.029305:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:3.0:1788204936.029309:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204936.029310:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029313:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029322:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204936.029324:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.029327:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204936.029328:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204936.029330:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204936.029333:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029341:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204936.029342:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.029345:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204936.029346:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204936.029349:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204936.029358:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490432:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204936.029362:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204936.029364:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204936.029365:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.029367:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204936.029370:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204936.029372:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204936.029376:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204936.029379:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204936.029383:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204936.029386:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543490432, portal 10 00000100:00000001:3.0:1788204936.029388:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204936.029389:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000040:3.0:1788204936.029397:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029403:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204936.029406:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204936.029408:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204936.029410:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543490432, offset 0 00000400:00000200:3.0:1788204936.029414:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204936.029419:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.029422:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204936.029425:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204936.029428:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204936.029430:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204936.029433:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204936.029436:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471180:471180:256:4294967295] 192.168.201.118@tcp LPNI seq info [471039:471039:8:4294967295] 00000400:00000200:3.0:1788204936.029440:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.029446:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204936.029450:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204936.029454:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204936.029458:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204936.029461:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204936.029465:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204936.029468:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788204936.029473:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204936.029475:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204936.029478:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204936.029481:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204936.029484:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204936.029486:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204936.029489:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204936.029492:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029500:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.029503:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029515:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.029517:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204936.029520:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204936.029524:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204936.029527:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204936.029529:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321294867 00000100:00000001:3.0:1788204936.029532:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321296133 00000100:00000001:3.0:1788204936.029535:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321294867 00000100:00000001:3.0:1788204936.029537:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321296133 00000100:00000001:3.0:1788204936.029539:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204936.029541:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.029543:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204936.029546:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204936/real 1788204936] req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204936.029554:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204936.029555:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204936.029557:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204936.029559:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204936.029561:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204936.029562:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204936.029565:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204936.029573:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029580:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204936.029582:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029583:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.029585:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204936.029588:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029595:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204936.029599:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029606:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204936.029609:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029610:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.029611:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029613:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204936.029616:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204936.029618:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204936.029621:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204936.029623:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204936.029626:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204936.029634:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490432:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204936.029638:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204936.029641:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029647:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204936.029648:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.029650:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204936.029652:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204936.029654:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204936.029657:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490560:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204936.029661:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204936.029663:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204936.029665:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.029667:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204936.029670:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204936.029672:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204936.029675:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204936.029678:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204936.029681:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204936.029684:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543490560, portal 10 00000100:00000001:3.0:1788204936.029686:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204936.029687:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 00000100:00000040:3.0:1788204936.029693:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029699:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204936.029702:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204936.029704:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204936.029707:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543490560, offset 0 00000400:00000200:3.0:1788204936.029711:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204936.029714:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.029718:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204936.029720:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204936.029723:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204936.029725:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204936.029727:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204936.029730:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471181:471181:256:4294967295] 192.168.201.118@tcp LPNI seq info [471040:471040:8:4294967295] 00000400:00000200:3.0:1788204936.029734:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.029740:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204936.029744:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204936.029747:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204936.029751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204936.029754:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204936.029758:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204936.029760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788204936.029764:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204936.029767:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204936.029769:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204936.029773:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204936.029776:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204936.029777:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204936.029779:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204936.029782:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029789:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.029792:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029799:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.029800:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204936.029803:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204936.029806:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204936.029809:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204936.029811:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321294868 00000100:00000001:3.0:1788204936.029813:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321296133 00000100:00000001:3.0:1788204936.029815:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321294868 00000100:00000001:3.0:1788204936.029818:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321296133 00000100:00000001:3.0:1788204936.029820:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204936.029822:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.029824:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204936.029827:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204936/real 1788204936] req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204936.029835:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204936.029836:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204936.029838:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204936.029841:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204936.029842:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204936.029844:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204936.029847:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204936.029854:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029859:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204936.029860:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029861:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.029862:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204936.029865:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029871:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204936.029874:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029879:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204936.029881:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029882:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.029883:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.029902:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204936.029904:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204936.029907:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204936.029910:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204936.029911:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204936.029914:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204936.029921:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490560:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204936.029924:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204936.029927:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029933:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204936.029934:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.029936:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204936.029937:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204936.029939:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204936.029941:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490688:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204936.029944:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204936.029946:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204936.029947:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.029949:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204936.029952:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204936.029954:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204936.029957:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204936.029959:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204936.029962:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204936.029964:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543490688, portal 4 00000100:00000001:3.0:1788204936.029966:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204936.029967:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331919360 : -109424377632256 : ffff9c7aa6d76a00) 00000100:00000040:3.0:1788204936.029971:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.029977:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204936.029979:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204936.029981:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204936.029983:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543490688, offset 0 00000400:00000200:3.0:1788204936.029985:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204936.029988:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.029991:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204936.029994:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204936.029997:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204936.029999:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204936.030001:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204936.030004:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471182:471182:256:4294967295] 192.168.201.118@tcp LPNI seq info [471041:471041:8:4294967295] 00000400:00000200:3.0:1788204936.030008:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.030013:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204936.030016:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204936.030019:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204936.030022:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204936.030025:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204936.030028:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204936.030030:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788204936.030034:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204936.030035:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204936.030038:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204936.030040:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204936.030043:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204936.030044:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204936.030046:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204936.030049:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030055:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.030058:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030063:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.030065:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204936.030067:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204936.030070:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204936.030073:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204936.030075:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321294877 00000100:00000001:3.0:1788204936.030078:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321296133 00000100:00000001:3.0:1788204936.030080:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321294877 00000100:00000001:3.0:1788204936.030082:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321296133 00000100:00000001:3.0:1788204936.030085:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204936.030087:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.030107:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204936.030112:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204936/real 1788204936] req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204936.030121:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204936.030122:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204936.030124:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204936.030127:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204936.030128:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204936.030130:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204936.030133:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204936.030140:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030146:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204936.030147:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030149:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.030150:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204936.030153:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030159:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204936.030163:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030169:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204936.030172:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030173:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.030175:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030176:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204936.030178:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204936.030182:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204936.030185:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204936.030187:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204936.030191:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204936.030199:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490688:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204936.030203:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204936.030206:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030213:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204936.030214:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.030216:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204936.030218:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204936.030220:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204936.030222:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490816:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204936.030225:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204936.030227:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204936.030229:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204936.030231:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204936.030235:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204936.030237:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204936.030240:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204936.030243:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204936.030246:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204936.030249:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543490816, portal 4 00000100:00000001:3.0:1788204936.030251:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204936.030252:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:3.0:1788204936.030258:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030263:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204936.030266:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204936.030268:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204936.030271:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543490816, offset 0 00000400:00000200:3.0:1788204936.030275:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204936.030278:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.030281:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204936.030284:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204936.030286:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204936.030289:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204936.030291:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204936.030294:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471183:471183:256:4294967295] 192.168.201.118@tcp LPNI seq info [471042:471042:8:4294967295] 00000400:00000200:3.0:1788204936.030297:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204936.030302:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204936.030307:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204936.030310:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204936.030313:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204936.030316:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788204936.030320:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204936.030322:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788204936.030326:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204936.030328:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204936.030331:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204936.030334:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204936.030337:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204936.030339:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204936.030341:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204936.030344:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030351:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.030353:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204961 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030359:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.030361:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204936.030364:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204936.030367:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204936.030370:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204936.030373:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321294901 00000100:00000001:3.0:1788204936.030375:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321296133 00000100:00000001:3.0:1788204936.030377:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321294901 00000100:00000001:3.0:1788204936.030380:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321296133 00000100:00000001:3.0:1788204936.030382:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204936.030384:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.030386:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204936.030389:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204936/real 1788204936] req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204936.030397:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204936.030398:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204936.030400:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204936.030402:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204936.030404:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204936.030406:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204936.030409:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204936.030417:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030423:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204936.030424:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030426:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.030427:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204936.030431:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030437:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204936.030440:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030447:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204936.030449:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030451:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204936.030452:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030454:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204936.030456:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204936.030459:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204936.030461:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204936.030464:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204936.030467:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204936.030474:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543490816:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204936.030478:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030480:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.030482:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543490432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030489:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204936.030490:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204936.030492:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204936.030495:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204936.030497:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204936.030498:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204936.030500:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204936.030502:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788204936.030505:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204936.030507:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204936.030508:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:3.0:1788204936.030511:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204936.030512:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030514:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.030517:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543490560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030523:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204936.030525:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204936.030526:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204936.030528:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204936.030529:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204936.030531:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204936.030533:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204936.030534:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:3.0:1788204936.030537:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204936.030538:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204936.030539:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:3.0:1788204936.030541:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204936.030543:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030544:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.030547:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76a00 x1875055543490688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030552:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204936.030554:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204936.030555:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204936.030557:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204936.030558:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204936.030559:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204936.030561:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204936.030562:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:3.0:1788204936.030565:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204936.030566:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204936.030567:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76a00. 00000100:00000001:3.0:1788204936.030569:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204936.030570:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030572:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204936.030575:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77b80 x1875055543490816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204961 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204936.030585:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204936.030586:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204936.030587:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204936.030590:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204936.030591:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204936.030592:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204936.030594:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204936.030595:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:3.0:1788204936.030597:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204936.030598:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204936.030599:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77b80. 00000100:00000001:3.0:1788204936.030601:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204936.030602:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030604:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.030608:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204936.030609:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.030611:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204936.030613:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204936.860228:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204936.860228:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204936.860234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204936.860240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204936.860240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204936.860244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204936.860246:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204936.860249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204936.860250:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204936.860251:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204936.860255:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204936.860260:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204936.860262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204936.860265:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204936.860266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204936.860266:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.860271:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204936.860272:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204936.860274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204936.860276:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204936.860277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204936.860277:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204936.860282:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204936.860283:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204936.860286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204936.860289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204936.860363:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204936.860368:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204936.860372:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204937.372144:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321296133, at 4321296192 00000100:00000040:3.0:1788204937.372154:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321296133, at 4321296192 00000100:00000010:2.0:1788204937.372159:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc900. 00000100:00000010:3.0:1788204937.372170:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc240. 00000100:00000001:3.0:1788204937.884192:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204937.884200:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204937.884202:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204937.884204:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204937.884207:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204937.884212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204937.884216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204937.884217:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204937.884218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204937.884224:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204937.884225:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204937.884226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204937.884235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204937.884235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204937.884235:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204937.884240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204937.884241:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204937.884243:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204937.884244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204937.884245:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204937.884251:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204937.884254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204937.884255:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204937.884263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204937.884263:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204937.884278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204937.884334:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204937.884339:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204937.884342:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204938.908214:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204938.908221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204938.908226:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204938.908229:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788204938.908230:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204938.908233:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788204938.908233:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204938.908239:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204938.908240:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204938.908240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204938.908244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204938.908247:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204938.908249:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204938.908253:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204938.908254:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204938.908257:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204938.908258:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204938.908260:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204938.908260:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204938.908263:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204938.908266:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204938.908268:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204938.908269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204938.908272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204938.908275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204938.908276:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204938.908278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204938.908280:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204938.908281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204938.908287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204938.908287:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204938.908290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204938.908427:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204938.908498:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204938.908512:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204939.932221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204939.932221:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204939.932226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204939.932232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204939.932232:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204939.932232:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204939.932236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204939.932242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204939.932243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204939.932244:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204939.932246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204939.932248:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204939.932249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204939.932251:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204939.932253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204939.932253:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204939.932254:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204939.932257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204939.932259:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204939.932259:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204939.932264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204939.932264:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204939.932264:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204939.932268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204939.932268:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204939.932268:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204939.932335:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204939.932339:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204939.932342:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204940.956219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204940.956223:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204940.956226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204940.956227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204940.956229:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204940.956231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204940.956231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204940.956233:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204940.956239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204940.956241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204940.956241:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204940.956243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204940.956245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204940.956247:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204940.956248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204940.956249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204940.956255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204940.956255:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204940.956255:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204940.956258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204940.956262:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204940.956262:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204940.956267:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204940.956267:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204940.956273:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204940.956273:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204940.956332:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204940.956337:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204940.956341:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204941.148254:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204941.148270:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204941.148273:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204941.148277:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204941.148283:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204941.148293:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26627 00000400:00000200:2.0:1788204941.148306:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204941.148315:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204941.148320:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204941.148324:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148327:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204941.148330:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148333:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204941.148338:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204941.148340:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204941.148344:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204941.148348:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204941.148351:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204941.148354:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204941.148357:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148359:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204941.148361:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148363:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204941.148367:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204941.148371:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148374:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204941.148377:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204941.148381:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204941.148385:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.148387:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204941.148390:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204941.148402:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204941.148406:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204941.148411:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204941.148413:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.148417:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204941.148420:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204941.148426:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204941.148435:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204941.148444:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204941.148455:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204941.148467:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204941.148473:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204941.148478:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204941.148509:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204941.148513:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204941.148517:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204941.148518:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204941.148520:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204941.148523:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204941.148532:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26627 00000400:00000200:2.0:1788204941.148536:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204941.148540:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204941.148543:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204941.148547:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148550:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204941.148553:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204941.148553:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204941.148559:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204941.148561:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788204941.148562:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204941.148566:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:3.0:1788204941.148567:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788204941.148568:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204941.148573:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204941.148577:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204941.148579:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204941.148580:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204941.148589:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148594:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204941.148598:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204941.148600:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204941.148604:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204941.148604:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204941.148610:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:3.0:1788204941.148610:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204941.148614:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788204941.148617:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788204941.148618:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.148624:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204941.148624:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543490944:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788204941.148630:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204941.148631:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204941.148637:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788204941.148640:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204941.148640:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.148642:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788204941.148646:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.148650:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000010:3.0:1788204941.148650:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204941.148654:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788204941.148658:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788204941.148662:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00000001:2.0:1788204941.148666:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204941.148667:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:2.0:1788204941.148672:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204941.148677:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788204941.148680:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204941.148683:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204941.148685:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.148686:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204941.148688:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:3.0:1788204941.148688:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 02000000:00000001:2.0:1788204941.148691:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204941.148697:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543490944, portal 25 02000000:00000010:2.0:1788204941.148700:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:3.0:1788204941.148701:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204941.148703:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331911296 : -109424377640320 : ffff9c7aa6d74a80) 00000100:00000001:2.0:1788204941.148708:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204941.148709:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204941.148719:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204941.148719:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204941.148726:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204941.148732:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000001:2.0:1788204941.148735:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788204941.148737:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543490944, offset 0 00000100:00000001:2.0:1788204941.148740:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204941.148747:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204941.148751:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148762:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204941.148766:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788204941.148766:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:2.0:1788204941.148772:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204941.148775:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204941.148776:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204941.148779:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204941.148783:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26627 00000400:00000200:3.0:1788204941.148790:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204941.148808:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204941.148814:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204941.148818:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204941.148821:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204941.148827:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471184:471184:256:4294967295] 192.168.201.118@tcp LPNI seq info [471043:471043:8:4294967295] 00000400:00000200:3.0:1788204941.148832:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204941.148843:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000400:00000200:2.0:1788204941.148852:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.0:1788204941.148854:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000400:00000200:2.0:1788204941.148859:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204941.148863:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000010:3.0:1788204941.148863:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000100:00000040:2.0:1788204941.148869:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148874:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788204941.148874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204941.148879:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000040:2.0:1788204941.148880:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148886:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788204941.148889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:2.0:1788204941.148891:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204941.148894:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000800:00000200:3.1:1788204941.148894:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000020:00000001:2.0:1788204941.148899:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000100:3.0:1788204941.148901:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000040:2.0:1788204941.148903:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000800:00000010:3.0:1788204941.148905:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000020:00000001:2.0:1788204941.148909:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204941.148913:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788204941.148914:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1788204941.148919:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204941.148923:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788204941.148924:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204941.148929:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000100:00000040:2.0:1788204941.148930:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:3.0:1788204941.148934:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:2.0:1788204941.148935:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204941.148939:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:2.0:1788204941.148940:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000200:3.0:1788204941.148946:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204941.148947:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204941.148953:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204941.148959:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204941.148960:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788204941.148963:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1788204941.148964:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204941.148970:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.148974:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.148975:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204941.148977:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:2.0:1788204941.148979:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204941.148983:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.148984:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.148986:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204941.148990:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:2.0:1788204941.148993:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000100:00000001:3.0:1788204941.148995:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321301253 00000020:00000040:2.0:1788204941.148998:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000100:00000010:3.0:1788204941.149001:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc240. 02000000:00000001:2.0:1788204941.149003:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204941.149006:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.149009:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204941.149009:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 02000000:00000001:2.0:1788204941.149012:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204941.149016:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149018:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321301253 02000000:00000010:2.0:1788204941.149022:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000010:3.0:1788204941.149023:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc4e0. 00000100:00000001:3.0:1788204941.149028:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1788204941.149029:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149030:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204941.149034:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149036:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788204941.149039:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788204941.149042:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204941/real 1788204941] req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204941.149070:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204941.149075:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788204941.149077:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204941.149079:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204941.149081:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00000040:2.0:1788204941.149082:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204941.149084:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204941.149087:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204941.149110:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204941.149114:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204941.149120:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204941.149138:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149138:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204941.149141:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204941.149143:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204941.149144:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.149147:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204941.149152:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788204941.149153:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204941.149158:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204941.149162:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204941.149163:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1788204941.149168:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204941.149170:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204941.149174:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204941.149177:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788204941.149179:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26627 00000100:00000001:3.0:1788204941.149179:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.149181:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.149184:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204941.149186:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1788204941.149189:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788204941.149191:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204941.149197:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000400:00000200:2.0:1788204941.149198:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204941.149200:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1788204941.149203:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204941.149206:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204941.149211:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00100000:3.0:1788204941.149216:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543490944:192.168.201.118@tcp:250:kworker.0 00000100:00000001:2.0:1788204941.149217:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204941.149222:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204941.149224:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204941.149225:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788204941.149230:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788204941.149230:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74a80 x1875055543490944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204941.149236:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204941.149238:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204941.149242:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788204941.149245:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000040:2.0:1788204941.149246:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 02000000:00000001:3.0:1788204941.149250:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1788204941.149251:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204941.149255:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 02000000:00000010:3.0:1788204941.149255:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204941.149261:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:2.0:1788204941.149264:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000020:00000001:3.0:1788204941.149265:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204941.149268:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000100:00000001:2.0:1788204941.149270:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1788204941.149274:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000100:00000040:2.0:1788204941.149275:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000010:3.0:1788204941.149278:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204941.149279:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204941.149285:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 02000000:00000001:3.0:1788204941.149287:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1788204941.149289:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788204941.149291:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788204941.149293:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204941.149295:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74a80. 02000000:00000001:2.0:1788204941.149296:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204941.149301:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204941.149302:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204941.149305:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204941.149306:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149309:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204941.149310:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204941.149315:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788204941.149317:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.149319:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204941.149323:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788204941.149327:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788204941.149329:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788204941.149330:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204941.149333:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204941.149334:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149338:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204941.149339:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204941.149340:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788204941.149343:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204941.149343:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788204941.149347:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204941.149349:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204941.149355:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:3.0:1788204941.149356:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204941.149358:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.149360:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788204941.149362:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.149362:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204941.149368:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204941.149371:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491072:192.168.201.118@tcp:38:kworker.0 00000100:00100000:2.0:1788204941.149374:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204941.149375:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204941.149378:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204941.149380:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.149383:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204941.149386:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204941.149390:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.149391:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204941.149394:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204941.149396:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204941.149397:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204941.149402:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000040:2.0:1788204941.149404:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204941.149405:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543491072, portal 10 00000100:00000001:3.0:1788204941.149409:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204941.149410:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913984 : -109424377637632 : ffff9c7aa6d75500) 00000100:00000040:3.0:1788204941.149418:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204941.149421:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149425:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788204941.149427:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1788204941.149428:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204941.149431:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204941.149434:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543491072, offset 0 00000100:00080000:2.0:1788204941.149437:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1788204941.149439:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204941.149441:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788204941.149444:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:2.0:1788204941.149446:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788204941.149449:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:2.0:1788204941.149452:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788204941.149453:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1788204941.149459:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26627 00000400:00000200:3.0:1788204941.149460:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204941.149464:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204941.149466:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204941.149468:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471185:471185:256:4294967295] 192.168.201.118@tcp LPNI seq info [471044:471044:8:4294967295] 00000400:00000200:3.0:1788204941.149471:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204941.149475:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204941.149477:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000400:00000200:2.0:1788204941.149481:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000010:3.0:1788204941.149482:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000400:00000200:2.0:1788204941.149490:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:3.0:1788204941.149490:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204941.149495:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00080000:2.0:1788204941.149496:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1788204941.149501:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788204941.149503:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:3.1:1788204941.149504:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788204941.149508:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1788204941.149510:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204941.149513:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000100:00000040:2.0:1788204941.149514:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204941.149516:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788204941.149518:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204941.149522:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1788204941.149524:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204941.149527:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000400:00000200:3.0:1788204941.149527:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000020:00000001:2.0:1788204941.149531:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:3.0:1788204941.149532:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000020:00000040:2.0:1788204941.149535:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:3.0:1788204941.149537:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1788204941.149540:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204941.149544:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000200:3.0:1788204941.149544:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204941.149552:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204941.149557:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204941.149562:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204941.149564:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788204941.149566:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204941.149569:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000040:3.0:1788204941.149571:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204941.149575:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204941.149582:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204941.149587:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204941.149588:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204941.149591:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204941.149592:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1788204941.149596:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149598:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204941.149599:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204941.149602:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204941.149606:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204941.149606:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000100:00000001:2.0:1788204941.149611:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204941.149614:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204941.149616:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321299987 00000100:00000010:2.0:1788204941.149617:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000100:00000001:3.0:1788204941.149619:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321301253 00000020:00000040:2.0:1788204941.149623:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788204941.149623:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321299987 00000100:00000001:3.0:1788204941.149627:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321301253 02000000:00000001:2.0:1788204941.149629:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788204941.149630:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204941.149632:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149632:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149636:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788204941.149638:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788204941.149641:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204941/real 1788204941] req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204941.149642:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204941.149646:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788204941.149652:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:2.0:1788204941.149653:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000400:00000001:3.0:1788204941.149654:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204941.149657:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00000001:2.0:1788204941.149660:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204941.149661:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204941.149664:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204941.149666:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:2.0:1788204941.149672:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788204941.149673:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204941.149691:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204941.149698:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204941.149700:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788204941.149703:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149703:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.149708:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.149709:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204941.149713:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204941.149717:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204941.149735:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149737:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204941.149741:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26637) 00000100:00000040:3.0:1788204941.149745:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.149753:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204941.149756:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.149758:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.149759:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.149760:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204941.149762:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204941.149764:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204941.149767:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204941.149769:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204941.149771:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204941.149778:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491072:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204941.149783:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204941.149786:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.149792:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204941.149793:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.149795:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204941.149807:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204941.149810:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204941.149813:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491200:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204941.149817:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204941.149819:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204941.149820:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.149822:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204941.149826:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204941.149828:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204941.149831:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204941.149833:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204941.149836:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204941.149838:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543491200, portal 10 00000100:00000001:3.0:1788204941.149840:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204941.149841:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331909504 : -109424377642112 : ffff9c7aa6d74380) 00000100:00000040:3.0:1788204941.149845:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.149850:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204941.149851:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204941.149852:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204941.149854:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543491200, offset 0 00000400:00000200:3.0:1788204941.149856:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204941.149858:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204941.149861:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204941.149862:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204941.149865:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204941.149866:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204941.149868:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204941.149870:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471186:471186:256:4294967295] 192.168.201.118@tcp LPNI seq info [471045:471045:8:4294967295] 00000400:00000200:3.0:1788204941.149872:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204941.149876:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204941.149878:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204941.149880:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204941.149882:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204941.149884:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204941.149887:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204941.149889:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204941.149891:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204941.149892:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204941.149894:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204941.149904:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204941.149908:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204941.149910:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204941.149911:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204941.149914:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.149919:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204941.149921:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.149925:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149926:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204941.149928:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204941.149930:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204941.149932:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204941.149933:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321299988 00000100:00000001:3.0:1788204941.149935:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321301253 00000100:00000001:3.0:1788204941.149937:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321299988 00000100:00000001:3.0:1788204941.149938:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321301253 00000100:00000001:3.0:1788204941.149940:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204941.149941:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.149943:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204941.149945:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204941/real 1788204941] req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204941.149949:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204941.149950:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204941.149951:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204941.149952:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204941.149954:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204941.149955:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204941.149956:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204941.149961:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.149965:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204941.149965:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.149966:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.149967:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204941.149969:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.149972:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204941.149974:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.149978:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204941.149979:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.149980:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.149981:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.149983:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204941.149985:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204941.149986:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204941.149989:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204941.149990:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204941.149992:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204941.149997:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491200:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204941.149999:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150000:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204941.150002:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75500 x1875055543491072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150005:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204941.150006:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204941.150007:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204941.150008:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204941.150009:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204941.150010:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204941.150011:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204941.150012:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:3.0:1788204941.150014:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204941.150014:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204941.150015:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75500. 00000100:00000001:3.0:1788204941.150017:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204941.150017:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150019:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204941.150020:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74380 x1875055543491200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150025:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204941.150025:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204941.150026:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204941.150028:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204941.150029:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204941.150029:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204941.150031:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204941.150032:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:3.0:1788204941.150033:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204941.150034:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204941.150034:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74380. 00000100:00000001:3.0:1788204941.150035:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204941.150036:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150037:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150039:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204941.150040:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.150041:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204941.150042:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204941.150043:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204941.150045:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150048:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204941.150049:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.150050:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204941.150051:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204941.150053:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204941.150054:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491328:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204941.150056:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204941.150057:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204941.150058:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.150059:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204941.150061:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204941.150063:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204941.150065:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204941.150066:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204941.150068:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204941.150070:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543491328, portal 4 00000100:00000001:3.0:1788204941.150071:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204941.150073:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331911296 : -109424377640320 : ffff9c7aa6d74a80) 00000100:00000040:3.0:1788204941.150076:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150081:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204941.150082:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204941.150083:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204941.150085:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543491328, offset 0 00000400:00000200:3.0:1788204941.150087:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204941.150105:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204941.150107:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204941.150108:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204941.150110:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204941.150111:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204941.150113:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204941.150115:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471187:471187:256:4294967295] 192.168.201.118@tcp LPNI seq info [471046:471046:8:4294967295] 00000400:00000200:3.0:1788204941.150118:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204941.150121:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204941.150125:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204941.150127:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204941.150129:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204941.150131:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204941.150134:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204941.150135:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204941.150137:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204941.150139:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204941.150141:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204941.150143:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204941.150144:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204941.150146:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204941.150147:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204941.150149:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150156:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204941.150159:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150165:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.150167:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204941.150170:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204941.150173:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204941.150176:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204941.150178:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321299998 00000100:00000001:3.0:1788204941.150180:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321301253 00000100:00000001:3.0:1788204941.150183:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321299998 00000100:00000001:3.0:1788204941.150185:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321301253 00000100:00000001:3.0:1788204941.150187:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204941.150189:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.150191:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204941.150195:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204941/real 1788204941] req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204941.150203:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204941.150204:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204941.150206:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204941.150208:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204941.150209:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204941.150210:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204941.150212:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204941.150218:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150222:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204941.150223:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150224:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.150224:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204941.150227:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150231:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204941.150234:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150241:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204941.150243:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150245:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.150246:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150247:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204941.150249:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204941.150252:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204941.150255:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204941.150258:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204941.150261:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204941.150269:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491328:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204941.150273:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204941.150276:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150283:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204941.150284:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.150286:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204941.150287:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204941.150289:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204941.150292:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491456:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204941.150295:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204941.150297:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204941.150299:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204941.150300:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204941.150303:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204941.150305:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204941.150308:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204941.150311:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204941.150314:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204941.150317:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543491456, portal 4 00000100:00000001:3.0:1788204941.150319:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204941.150320:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:3.0:1788204941.150327:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150333:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204941.150335:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204941.150337:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204941.150340:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543491456, offset 0 00000400:00000200:3.0:1788204941.150344:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204941.150347:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204941.150351:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204941.150353:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204941.150355:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204941.150358:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204941.150360:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204941.150363:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471188:471188:256:4294967295] 192.168.201.118@tcp LPNI seq info [471047:471047:8:4294967295] 00000400:00000200:3.0:1788204941.150366:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204941.150371:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204941.150375:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204941.150379:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204941.150382:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204941.150385:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788204941.150389:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204941.150391:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788204941.150394:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204941.150396:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204941.150399:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204941.150402:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204941.150405:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204941.150406:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204941.150409:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204941.150412:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150420:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204941.150423:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204966 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150428:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.150429:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204941.150431:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204941.150433:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204941.150435:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204941.150437:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321300021 00000100:00000001:3.0:1788204941.150439:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321301253 00000100:00000001:3.0:1788204941.150440:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321300021 00000100:00000001:3.0:1788204941.150441:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321301253 00000100:00000001:3.0:1788204941.150443:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204941.150444:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.150445:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204941.150447:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204941/real 1788204941] req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204941.150452:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204941.150453:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204941.150454:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204941.150455:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204941.150456:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204941.150458:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204941.150460:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204941.150465:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150469:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204941.150470:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150470:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.150471:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204941.150473:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150476:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204941.150478:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150482:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204941.150484:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150485:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204941.150485:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150487:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204941.150488:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204941.150489:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204941.150491:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204941.150492:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204941.150494:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204941.150498:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491456:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204941.150501:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150502:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204941.150505:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74a80 x1875055543491328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150509:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204941.150510:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204941.150511:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204941.150513:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204941.150514:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204941.150514:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204941.150515:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204941.150517:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:3.0:1788204941.150519:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204941.150520:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204941.150521:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74a80. 00000100:00000001:3.0:1788204941.150522:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204941.150523:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150524:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204941.150526:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77b80 x1875055543491456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204966 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204941.150532:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204941.150533:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204941.150535:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204941.150537:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204941.150538:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204941.150539:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204941.150541:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204941.150542:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788204941.150545:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204941.150546:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204941.150547:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77b80. 00000100:00000001:3.0:1788204941.150550:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204941.150552:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150553:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204941.150557:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204941.150558:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.150560:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204941.150562:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.980180:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204941.980190:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204941.980192:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204941.980197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00100000:3.0:1788204941.980198:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204941.980206:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204941.980206:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204941.980207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204941.980210:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204941.980211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204941.980213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204941.980221:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1788204941.980222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204941.980224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204941.980227:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204941.980229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204941.980230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204941.980233:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204941.980237:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204941.980238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204941.980242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204941.980243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204941.980248:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204941.980249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204941.980253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204941.980253:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204941.980317:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204941.980321:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204941.980324:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204942.492220:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321301253, at 4321301312 00000100:00000040:3.0:1788204942.492226:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321301253, at 4321301312 00000100:00000010:2.0:1788204942.492238:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc240. 00000100:00000010:3.0:1788204942.492243:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc4e0. 00000100:00000001:1.0:1788204943.004223:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204943.004223:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204943.004227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204943.004232:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204943.004233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204943.004233:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204943.004238:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204943.004238:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204943.004242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204943.004248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204943.004250:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204943.004250:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204943.004252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204943.004259:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204943.004260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204943.004262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204943.004267:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204943.004270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204943.004270:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204943.004272:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204943.004273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204943.004275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204943.004283:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204943.004284:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204943.004289:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204943.004289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204943.004355:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204943.004368:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204943.004371:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204944.028140:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204944.028150:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204944.028151:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204944.028154:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204944.028155:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204944.028157:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204944.028160:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204944.028161:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204944.028166:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204944.028167:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204944.028170:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204944.028171:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204944.028175:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204944.028176:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204944.028178:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204944.028179:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204944.028184:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204944.028189:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204944.028223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204944.028233:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204944.028235:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204944.028239:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204944.028245:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028247:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204944.028255:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204944.028256:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204944.028257:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028259:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204944.028260:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028262:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204944.028268:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204944.028270:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204944.028271:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028272:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204944.028273:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204944.028277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204944.028340:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204944.028344:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204944.028346:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204945.052213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204945.052214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204945.052214:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204945.052222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204945.052224:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204945.052224:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204945.052231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204945.052231:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204945.052233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204945.052238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204945.052242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204945.052242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204945.052243:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204945.052249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204945.052255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204945.052255:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204945.052257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204945.052261:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204945.052264:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204945.052265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204945.052268:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204945.052269:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204945.052271:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204945.052278:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204945.052281:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204945.052285:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204945.052326:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204945.052331:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204945.052335:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204946.076195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204946.076195:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204946.076199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204946.076203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204946.076203:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204946.076204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204946.076207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.076207:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204946.076212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204946.076215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204946.076215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204946.076215:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204946.076221:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204946.076222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204946.076223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204946.076226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204946.076227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.076227:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204946.076230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204946.076233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204946.076234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.076235:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204946.076237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.076239:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204946.076242:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.076242:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204946.076295:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204946.076298:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204946.076300:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204946.268207:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204946.268216:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204946.268218:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204946.268220:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204946.268224:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204946.268230:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26632 00000400:00000200:0.0:1788204946.268238:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204946.268242:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204946.268245:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204946.268247:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268249:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204946.268251:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268253:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204946.268257:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204946.268259:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204946.268262:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204946.268265:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204946.268267:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204946.268270:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204946.268273:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268275:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204946.268277:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268279:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204946.268282:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204946.268285:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268288:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204946.268291:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204946.268294:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204946.268297:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.268298:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204946.268300:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204946.268307:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788204946.268310:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204946.268313:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204946.268314:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.268317:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204946.268319:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204946.268322:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204946.268329:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788204946.268334:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204946.268341:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204946.268348:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204946.268352:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204946.268355:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204946.268371:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204946.268373:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204946.268375:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204946.268376:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204946.268377:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204946.268379:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204946.268381:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26632 00000400:00000200:0.0:1788204946.268384:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204946.268386:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204946.268387:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204946.268389:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268391:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204946.268392:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268393:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204946.268395:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204946.268395:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204946.268397:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204946.268398:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204946.268400:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204946.268401:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204946.268403:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268405:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204946.268406:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268407:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204946.268409:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204946.268411:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268412:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204946.268413:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204946.268416:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788204946.268418:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204946.268421:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.268422:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204946.268423:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1788204946.268425:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204946.268428:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.268428:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788204946.268431:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204946.268434:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788204946.268436:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204946.268438:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204946.268439:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204946.268439:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204946.268441:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204946.268443:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204946.268445:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204946.268449:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788204946.268453:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.268455:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204946.268458:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204946.268459:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788204946.268463:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788204946.268468:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788204946.268468:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204946.268471:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.268473:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204946.268474:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204946.268481:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543491584:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788204946.268484:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204946.268488:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204946.268489:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788204946.268491:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204946.268493:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204946.268495:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204946.268497:0:399478:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1788204946.268497:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204946.268501:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204946.268502:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26632 00000400:00000200:0.0:1788204946.268506:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204946.268507:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204946.268508:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204946.268510:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268511:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204946.268512:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268514:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204946.268516:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204946.268516:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204946.268517:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204946.268518:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 02000000:00000001:3.0:1788204946.268518:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1788204946.268522:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204946.268523:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204946.268524:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268525:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:3.0:1788204946.268526:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 00000100:00000040:0.0:1788204946.268527:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:3.0:1788204946.268532:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.268533:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204946.268537:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204946.268539:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268541:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204946.268543:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788204946.268543:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:0.0:1788204946.268546:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204946.268548:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204946.268549:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:0.0:1788204946.268550:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204946.268552:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204946.268555:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.268556:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204946.268559:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000400:00000010:3.0:1788204946.268560:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000020:00000040:0.0:1788204946.268561:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204946.268564:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204946.268566:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.268567:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204946.268567:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204946.268569:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204946.268570:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543491584, portal 25 02000000:00000010:0.0:1788204946.268572:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:3.0:1788204946.268575:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788204946.268576:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.268578:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114944 : -109419805436672 : ffff9c7bb75d9500) 00000100:00100000:0.0:1788204946.268581:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204946.268586:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204946.268589:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204946.268593:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204946.268599:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.268614:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788204946.268616:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204946.268619:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:0.0:1788204946.268620:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204946.268623:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:3.0:1788204946.268623:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000001:0.0:1788204946.268626:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204946.268628:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1788204946.268628:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543491584, offset 0 00000100:00000001:0.0:1788204946.268632:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204946.268635:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26632 00000400:00000200:3.0:1788204946.268638:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:0.0:1788204946.268642:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204946.268644:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204946.268645:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788204946.268648:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:0.0:1788204946.268649:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268652:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204946.268653:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268656:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204946.268656:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:0.0:1788204946.268658:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204946.268660:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204946.268662:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1788204946.268663:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:0.0:1788204946.268664:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204946.268667:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788204946.268668:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000020:00000001:0.0:1788204946.268670:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788204946.268673:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1788204946.268675:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268678:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204946.268678:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000040:0.0:1788204946.268682:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268684:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204946.268687:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:3.0:1788204946.268687:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471189:471189:256:4294967295] 192.168.201.118@tcp LPNI seq info [471048:471048:8:4294967295] 00000100:00080000:0.0:1788204946.268692:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204946.268696:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204946.268696:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.268706:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:0.0:1788204946.268707:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:3.0:1788204946.268727:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.268732:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204946.268737:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204946.268739:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204946.268743:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204946.268747:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204946.268751:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.268754:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204946.268761:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204946.268766:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204946.268772:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204946.268775:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204946.268779:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204946.268784:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.268792:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.268796:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.268808:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.268810:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204946.268817:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204946.268820:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204946.268823:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321306373 00000100:00000010:3.0:1788204946.268827:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc4e0. 00000100:00000001:3.0:1788204946.268831:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 02000000:00000001:0.0:1788204946.268837:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204946.268837:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321306373 02000000:00000001:0.0:1788204946.268841:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204946.268842:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc7e0. 00000100:00000001:0.0:1788204946.268843:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204946.268846:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000008:00100000:0.0:1788204946.268847:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204946.268850:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.268851:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788204946.268852:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.268856:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204946.268858:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000100:00000001:3.0:1788204946.268859:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000020:00000040:0.0:1788204946.268862:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204946.268866:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788204946.268866:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204946/real 1788204946] req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788204946.268868:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.268872:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204946.268874:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204946.268876:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204946.268880:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000400:00000001:3.0:1788204946.268880:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204946.268883:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788204946.268885:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204946.268886:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204946.268890:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204946.268893:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00100000:0.0:1788204946.268894:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204946.268895:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204946.268901:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204946.268903:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204946.268906:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204946.268912:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204946.268914:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204946.268929:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.268931:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1788204946.268932:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204946.268933:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204946.268936:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204946.268937:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788204946.268939:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204946.268940:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788204946.268941:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788204946.268945:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204946.268946:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204946.268950:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26632 00000400:00000200:0.0:1788204946.268957:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204946.268959:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788204946.268961:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204946.268964:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204946.268965:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.268971:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1788204946.268973:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204946.268974:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788204946.268976:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204946.268977:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.268979:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788204946.268981:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204946.268982:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788204946.268983:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204946.268985:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1788204946.268986:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204946.268988:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00080000:3.0:1788204946.268990:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:0.0:1788204946.268991:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204946.268994:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204946.268997:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00080000:3.0:1788204946.268997:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000020:00000001:0.0:1788204946.269000:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788204946.269002:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1788204946.269004:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.269006:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204946.269007:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204946.269008:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204946.269009:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204946.269010:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204946.269012:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204946.269015:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204946.269016:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204946.269019:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204946.269019:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204946.269023:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204946.269025:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204946.269029:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.269031:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204946.269035:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000100:00100000:3.0:1788204946.269037:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543491584:192.168.201.118@tcp:250:kworker.0 00000020:00000040:0.0:1788204946.269038:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204946.269041:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204946.269043:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204946.269046:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204946.269047:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204946.269048:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269049:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1788204946.269051:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:3.0:1788204946.269054:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788204946.269055:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204946.269062:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204946.269062:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9500 x1875055543491584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204946.269078:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204946.269081:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269086:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:0.0:1788204946.269087:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204946.269112:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269115:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204946.269119:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204946.269122:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000001:0.0:1788204946.269123:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788204946.269125:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000100:00000040:0.0:1788204946.269127:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26642) 00000020:00000001:3.0:1788204946.269128:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204946.269131:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2ac00. 02000000:00000001:3.0:1788204946.269135:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204946.269137:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204946.269138:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9500. 00000100:00000001:3.0:1788204946.269140:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204946.269142:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269143:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269149:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204946.269150:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269152:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204946.269153:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204946.269154:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204946.269157:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269161:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204946.269162:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.269164:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204946.269165:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204946.269166:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204946.269168:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491712:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204946.269170:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204946.269171:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204946.269172:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.269173:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269175:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204946.269176:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204946.269179:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204946.269181:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204946.269183:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204946.269185:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543491712, portal 10 00000100:00000001:3.0:1788204946.269186:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204946.269187:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904113152 : -109419805438464 : ffff9c7bb75d8e00) 00000100:00000040:3.0:1788204946.269192:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269196:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204946.269197:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204946.269199:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204946.269200:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543491712, offset 0 00000400:00000200:3.0:1788204946.269203:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204946.269206:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.269208:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204946.269210:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204946.269212:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204946.269214:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204946.269215:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204946.269217:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471190:471190:256:4294967295] 192.168.201.118@tcp LPNI seq info [471049:471049:8:4294967295] 00000400:00000200:3.0:1788204946.269219:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.269223:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204946.269227:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.269228:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204946.269231:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204946.269232:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204946.269235:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204946.269237:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204946.269239:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.269241:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204946.269242:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204946.269244:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204946.269246:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204946.269247:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204946.269249:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204946.269251:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269256:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.269257:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269269:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269271:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204946.269314:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204946.269317:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204946.269319:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204946.269320:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321305107 00000100:00000001:3.0:1788204946.269322:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321306373 00000100:00000001:3.0:1788204946.269323:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321305107 00000100:00000001:3.0:1788204946.269325:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321306373 00000100:00000001:3.0:1788204946.269326:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204946.269327:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269328:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204946.269330:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204946/real 1788204946] req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204946.269335:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204946.269336:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204946.269337:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204946.269339:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204946.269340:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204946.269341:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204946.269342:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204946.269348:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269351:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204946.269352:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269353:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.269354:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204946.269356:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269359:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204946.269361:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269366:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204946.269367:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269368:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.269369:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269370:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204946.269371:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204946.269373:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204946.269374:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204946.269376:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204946.269378:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204946.269382:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491712:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204946.269384:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204946.269386:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269390:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204946.269390:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.269392:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204946.269393:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204946.269394:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204946.269395:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491840:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204946.269397:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204946.269398:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204946.269399:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.269401:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269403:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204946.269404:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204946.269406:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204946.269407:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204946.269409:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204946.269410:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543491840, portal 10 00000100:00000001:3.0:1788204946.269412:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204946.269412:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904123904 : -109419805427712 : ffff9c7bb75db800) 00000100:00000040:3.0:1788204946.269416:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269419:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204946.269421:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204946.269422:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204946.269423:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543491840, offset 0 00000400:00000200:3.0:1788204946.269425:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204946.269427:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.269429:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204946.269431:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204946.269432:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204946.269434:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204946.269435:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204946.269437:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471191:471191:256:4294967295] 192.168.201.118@tcp LPNI seq info [471050:471050:8:4294967295] 00000400:00000200:3.0:1788204946.269440:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.269443:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204946.269446:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.269447:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204946.269450:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204946.269451:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204946.269454:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204946.269455:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204946.269457:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.269458:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204946.269460:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204946.269462:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204946.269464:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204946.269465:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204946.269467:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204946.269468:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269473:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.269475:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269478:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269479:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204946.269481:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204946.269482:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204946.269484:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204946.269485:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321305108 00000100:00000001:3.0:1788204946.269487:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321306373 00000100:00000001:3.0:1788204946.269488:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321305108 00000100:00000001:3.0:1788204946.269489:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321306373 00000100:00000001:3.0:1788204946.269490:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204946.269491:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269492:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204946.269494:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204946/real 1788204946] req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204946.269498:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204946.269499:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204946.269500:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204946.269501:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204946.269502:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204946.269503:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204946.269505:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204946.269509:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269513:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204946.269514:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269515:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.269515:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204946.269517:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269521:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204946.269523:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269526:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204946.269527:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269528:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.269529:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269530:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204946.269531:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204946.269532:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204946.269534:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204946.269535:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204946.269537:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204946.269541:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491840:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204946.269543:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204946.269544:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269548:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204946.269549:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.269551:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204946.269552:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204946.269553:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204946.269555:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491968:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204946.269556:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204946.269557:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204946.269558:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.269559:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269561:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204946.269562:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204946.269564:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204946.269565:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204946.269567:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204946.269568:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543491968, portal 4 00000100:00000001:3.0:1788204946.269569:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204946.269570:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114048 : -109419805437568 : ffff9c7bb75d9180) 00000100:00000040:3.0:1788204946.269573:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269577:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204946.269578:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204946.269579:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204946.269580:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543491968, offset 0 00000400:00000200:3.0:1788204946.269582:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204946.269584:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.269586:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204946.269588:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204946.269589:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204946.269590:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204946.269592:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204946.269593:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471192:471192:256:4294967295] 192.168.201.118@tcp LPNI seq info [471051:471051:8:4294967295] 00000400:00000200:3.0:1788204946.269596:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.269599:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204946.269602:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.269603:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204946.269605:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204946.269607:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204946.269609:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204946.269610:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204946.269613:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.269614:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204946.269616:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204946.269617:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204946.269619:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204946.269620:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204946.269621:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204946.269623:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269627:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.269629:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269632:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269633:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204946.269635:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204946.269636:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204946.269638:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204946.269639:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321305117 00000100:00000001:3.0:1788204946.269640:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321306373 00000100:00000001:3.0:1788204946.269642:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321305117 00000100:00000001:3.0:1788204946.269643:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321306373 00000100:00000001:3.0:1788204946.269644:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204946.269645:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269646:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204946.269647:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204946/real 1788204946] req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204946.269652:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204946.269653:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204946.269654:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204946.269655:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204946.269656:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204946.269657:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204946.269659:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204946.269663:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269666:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204946.269667:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269668:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.269669:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204946.269671:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269674:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204946.269676:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269679:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204946.269681:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269681:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.269682:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269683:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204946.269684:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204946.269685:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204946.269687:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204946.269688:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204946.269690:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204946.269695:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543491968:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204946.269697:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204946.269698:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269702:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204946.269702:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.269703:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204946.269704:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204946.269705:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204946.269707:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492096:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204946.269721:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204946.269722:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204946.269723:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204946.269724:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269726:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204946.269727:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204946.269729:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204946.269731:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204946.269733:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204946.269735:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543492096, portal 4 00000100:00000001:3.0:1788204946.269736:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204946.269737:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904118528 : -109419805433088 : ffff9c7bb75da300) 00000100:00000040:3.0:1788204946.269741:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269744:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204946.269746:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204946.269747:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204946.269748:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543492096, offset 0 00000400:00000200:3.0:1788204946.269751:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204946.269753:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.269755:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204946.269757:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204946.269758:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204946.269760:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204946.269761:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204946.269763:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471193:471193:256:4294967295] 192.168.201.118@tcp LPNI seq info [471052:471052:8:4294967295] 00000400:00000200:3.0:1788204946.269766:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204946.269769:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204946.269772:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.269773:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204946.269775:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204946.269777:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788204946.269779:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204946.269781:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788204946.269783:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204946.269784:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204946.269786:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204946.269788:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204946.269790:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204946.269791:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204946.269792:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204946.269794:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269798:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.269800:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204971 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269804:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269805:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204946.269806:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204946.269808:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204946.269809:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204946.269811:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321305140 00000100:00000001:3.0:1788204946.269812:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321306373 00000100:00000001:3.0:1788204946.269814:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321305140 00000100:00000001:3.0:1788204946.269815:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321306373 00000100:00000001:3.0:1788204946.269816:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204946.269817:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269818:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204946.269820:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204946/real 1788204946] req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204946.269824:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204946.269825:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204946.269826:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204946.269827:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204946.269829:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204946.269830:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204946.269831:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204946.269836:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269839:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204946.269840:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269841:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.269842:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204946.269843:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269847:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204946.269849:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269852:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204946.269854:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269854:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204946.269855:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269856:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204946.269858:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204946.269859:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204946.269860:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204946.269862:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204946.269864:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204946.269867:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492096:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204946.269869:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269870:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.269872:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8e00 x1875055543491712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269876:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204946.269877:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269878:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204946.269879:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204946.269880:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204946.269881:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204946.269882:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204946.269883:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a800. 02000000:00000001:3.0:1788204946.269885:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204946.269886:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204946.269887:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8e00. 00000100:00000001:3.0:1788204946.269888:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204946.269889:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269890:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.269892:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db800 x1875055543491840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269899:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204946.269900:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269901:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204946.269902:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204946.269902:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204946.269903:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204946.269904:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204946.269905:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:3.0:1788204946.269906:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204946.269906:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204946.269907:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db800. 00000100:00000001:3.0:1788204946.269908:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204946.269909:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269910:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.269911:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9180 x1875055543491968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269914:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204946.269915:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269916:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204946.269917:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204946.269917:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204946.269918:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204946.269919:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204946.269920:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:3.0:1788204946.269921:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204946.269921:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204946.269922:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9180. 00000100:00000001:3.0:1788204946.269923:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204946.269924:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269924:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204946.269926:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da300 x1875055543492096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204971 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204946.269929:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204946.269929:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204946.269931:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204946.269932:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204946.269932:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204946.269933:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204946.269934:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204946.269935:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29400. 02000000:00000001:3.0:1788204946.269936:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204946.269937:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204946.269937:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da300. 00000100:00000001:3.0:1788204946.269939:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204946.269939:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269940:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204946.269942:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204946.269943:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204946.269944:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204946.269945:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204947.100181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204947.100181:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204947.100185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204947.100189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204947.100189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204947.100189:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204947.100191:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204947.100193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204947.100196:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204947.100197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204947.100197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204947.100200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204947.100203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204947.100203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204947.100203:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204947.100206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204947.100207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204947.100207:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204947.100210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204947.100212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204947.100212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204947.100212:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204947.100216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204947.100216:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204947.100219:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204947.100221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204947.100260:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204947.100263:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204947.100266:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204947.612197:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321306373, at 4321306432 00000100:00000040:3.0:1788204947.612197:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321306373, at 4321306432 00000100:00000010:3.0:1788204947.612211:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc7e0. 00000100:00000010:2.0:1788204947.612215:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc4e0. 00000100:00000001:1.0:1788204948.124186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204948.124186:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204948.124191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204948.124193:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204948.124197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204948.124199:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204948.124206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204948.124207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204948.124208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204948.124208:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204948.124215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204948.124216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204948.124216:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204948.124217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204948.124221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204948.124223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204948.124223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204948.124224:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204948.124230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204948.124230:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204948.124236:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204948.124237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204948.124242:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204948.124242:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204948.124245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204948.124247:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204948.124284:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204948.124288:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204948.124292:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204949.148189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204949.148189:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204949.148196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204949.148197:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204949.148199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.148200:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204949.148203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204949.148206:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204949.148207:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204949.148209:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204949.148213:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788204949.148213:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204949.148213:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204949.148217:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204949.148218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204949.148219:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204949.148223:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204949.148224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204949.148225:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788204949.148228:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204949.148229:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204949.148229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204949.148234:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204949.148236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204949.148236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204949.148242:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204949.148247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204949.148248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204949.148251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204949.148253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204949.148256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204949.148258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204949.148340:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204949.148343:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204949.148345:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204950.172192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204950.172192:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204950.172196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204950.172200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204950.172201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204950.172202:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204950.172204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204950.172205:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204950.172206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204950.172209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204950.172211:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204950.172215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204950.172215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204950.172217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204950.172217:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204950.172218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204950.172222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204950.172223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204950.172223:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204950.172224:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204950.172225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204950.172227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204950.172229:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204950.172230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204950.172238:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204950.172241:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204950.172278:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204950.172281:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204950.172283:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204951.196170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204951.196170:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204951.196177:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204951.196178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204951.196179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204951.196181:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204951.196182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.196182:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204951.196187:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204951.196189:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204951.196190:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204951.196193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204951.196195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204951.196195:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204951.196196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.196200:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204951.196200:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204951.196202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204951.196204:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204951.196205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.196206:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204951.196208:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204951.196210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204951.196211:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204951.196212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.196215:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204951.196271:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204951.196275:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204951.196278:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204951.388202:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204951.388210:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204951.388212:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204951.388216:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204951.388220:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204951.388227:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26637 00000400:00000200:3.0:1788204951.388236:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204951.388242:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204951.388245:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204951.388248:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388252:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204951.388254:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388257:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204951.388261:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204951.388262:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788204951.388266:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204951.388269:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788204951.388271:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788204951.388273:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788204951.388276:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388278:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204951.388280:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388282:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204951.388285:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788204951.388287:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388290:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204951.388293:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204951.388296:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204951.388298:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388301:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204951.388304:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204951.388312:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788204951.388316:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788204951.388319:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204951.388321:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388325:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204951.388327:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204951.388331:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204951.388338:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412400. 00000100:00000001:3.0:1788204951.388344:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204951.388353:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204951.388369:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204951.388373:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204951.388378:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204951.388401:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204951.388405:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204951.388413:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204951.388414:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204951.388416:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204951.388419:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204951.388422:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26637 00000400:00000200:3.0:1788204951.388426:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204951.388431:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204951.388433:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204951.388436:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388438:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204951.388441:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1788204951.388441:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388443:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204951.388448:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788204951.388448:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204951.388450:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:1.0:1788204951.388452:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788204951.388454:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204951.388457:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788204951.388461:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788204951.388462:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204951.388464:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204951.388472:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388477:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204951.388481:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204951.388482:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204951.388486:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204951.388487:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788204951.388491:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:1.0:1788204951.388494:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788204951.388496:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:1.0:1788204951.388498:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204951.388501:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.388503:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204951.388505:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788204951.388510:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543492224:192.168.201.118@tcp:250:kworker.0 02000000:00000001:3.0:1788204951.388512:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204951.388518:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204951.388518:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388522:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788204951.388524:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:3.0:1788204951.388526:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788204951.388528:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788204951.388531:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.388533:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788204951.388535:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204951.388539:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 00000100:00000010:3.0:1788204951.388541:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 02000000:00000001:1.0:1788204951.388543:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788204951.388546:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000400:00000200:1.0:1788204951.388550:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788204951.388551:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1788204951.388554:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 02000000:00000001:3.0:1788204951.388554:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388558:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204951.388561:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1788204951.388563:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 02000000:00000001:3.0:1788204951.388563:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204951.388570:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413800. 00000100:00000200:1.0:1788204951.388571:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543492224, portal 25 00000100:00000001:3.0:1788204951.388573:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.388575:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204951.388577:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00100000:3.0:1788204951.388580:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1788204951.388585:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204951.388591:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204951.388595:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204951.388599:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.388610:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788204951.388611:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204951.388615:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204951.388634:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204951.388638:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543492224, offset 0 00000400:00000200:1.0:1788204951.388647:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788204951.388648:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204951.388653:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788204951.388655:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1788204951.388657:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788204951.388658:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204951.388662:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788204951.388664:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00080000:3.0:1788204951.388669:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26637 00000400:00000200:1.0:1788204951.388672:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204951.388676:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.388677:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204951.388684:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:1.0:1788204951.388685:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1788204951.388690:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1788204951.388691:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000040:3.0:1788204951.388698:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788204951.388700:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471194:471194:256:4294967295] 192.168.201.118@tcp LPNI seq info [471053:471053:8:4294967295] 00000400:00000200:1.0:1788204951.388706:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204951.388709:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204951.388714:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388719:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788204951.388720:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000020:00000001:3.0:1788204951.388723:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204951.388726:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788204951.388729:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.0:1788204951.388731:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000020:00000040:3.0:1788204951.388733:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788204951.388737:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000800:00000010:1.0:1788204951.388739:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000020:00000001:3.0:1788204951.388740:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788204951.388744:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388745:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204951.388746:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388747:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204951.388750:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788204951.388753:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388754:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204951.388756:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204951.388759:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1788204951.388760:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:3.0:1788204951.388761:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388765:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:1.0:1788204951.388767:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000008:00100000:3.0:1788204951.388768:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204951.388770:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388774:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204951.388777:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000800:00000200:1.1:1788204951.388778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:3.0:1788204951.388780:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000800:00000200:1.1:1788204951.388784:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 02000000:00000001:3.0:1788204951.388790:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000800:00000100:1.0:1788204951.388793:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788204951.388793:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388794:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204951.388798:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:1.0:1788204951.388799:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 02000000:00000001:3.0:1788204951.388799:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204951.388804:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:3.0:1788204951.388807:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204951.388808:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1788204951.388812:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204951.388817:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204951.388819:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204951.388823:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204951.388831:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788204951.388831:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204951.388835:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204951.388837:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204951.388838:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1788204951.388840:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000100:00080000:3.0:1788204951.388840:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204951.388845:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1788204951.388846:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00080000:3.0:1788204951.388848:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26637 00000100:00000001:1.0:1788204951.388852:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1788204951.388853:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204951.388857:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204951.388858:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1788204951.388861:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204951.388862:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388865:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204951.388868:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.388873:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204951.388876:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204951.388878:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788204951.388880:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204951.388882:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00000001:1.0:1788204951.388884:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788204951.388884:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788204951.388888:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788204951.388890:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204951.388896:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204951.388897:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388900:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204951.388901:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1788204951.388907:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204951.388910:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388910:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204951.388916:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:1.0:1788204951.388920:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788204951.388922:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204951.388926:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321311493 00000100:00000001:3.0:1788204951.388926:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204951.388931:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:1.0:1788204951.388935:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647ba0. 02000000:00000001:3.0:1788204951.388935:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204951.388938:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388942:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204951.388945:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000008:00100000:3.0:1788204951.388945:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788204951.388951:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388952:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.388955:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321311493 00000100:00000010:3.0:1788204951.388957:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000100:00000010:1.0:1788204951.388961:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647e40. 00000020:00000040:3.0:1788204951.388961:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204951.388964:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788204951.388967:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:3.0:1788204951.388967:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.388972:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788204951.388973:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204951.388973:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204951.388976:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.388979:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.388983:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000010:3.0:1788204951.388983:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00100000:1.0:1788204951.388990:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204951/real 1788204951] req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204951.389007:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788204951.389008:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788204951.389010:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204951.389014:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000100:00100000:3.0:1788204951.389015:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204951.389018:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204951.389023:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788204951.389023:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204951.389027:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389029:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1788204951.389031:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204951.389038:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204951.389039:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204951.389043:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204951.389050:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204951.389052:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204951.389055:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1788204951.389058:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204951.389058:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204951.389062:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26637 00000400:00000200:3.0:1788204951.389067:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204951.389071:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204951.389074:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204951.389077:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1788204951.389077:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.389081:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204951.389082:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204951.389082:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.389086:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204951.389088:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1788204951.389088:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204951.389123:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204951.389125:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204951.389128:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788204951.389129:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788204951.389131:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788204951.389135:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00100000:1.0:1788204951.389137:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204951.389141:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204951.389146:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204951.389151:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204951.389152:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204951.389156:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788204951.389160:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788204951.389161:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:1.0:1788204951.389170:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204951.389170:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204951.389173:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204951.389176:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389177:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.389181:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788204951.389181:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788204951.389186:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1788204951.389186:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204951.389189:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788204951.389191:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204951.389195:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788204951.389195:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204951.389200:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1788204951.389202:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000008:00000001:3.0:1788204951.389205:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389206:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788204951.389211:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204951.389215:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788204951.389218:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788204951.389222:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204951.389227:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204951.389229:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204951.389233:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1788204951.389234:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543492224:192.168.201.118@tcp:250:kworker.0 02000000:00000001:3.0:1788204951.389236:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204951.389239:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389242:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1788204951.389245:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:1.0:1788204951.389246:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.389252:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543492224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204951.389252:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204951.389261:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204951.389266:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204951.389271:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788204951.389274:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:1.0:1788204951.389275:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 00000100:00000001:3.0:1788204951.389278:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389281:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204951.389285:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000040:3.0:1788204951.389286:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788204951.389289:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204951.389294:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204951.389298:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 00000100:00000001:3.0:1788204951.389300:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389303:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204951.389307:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:3.0:1788204951.389307:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26647) 00000100:00000010:1.0:1788204951.389310:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788204951.389315:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204951.389317:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389319:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389326:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204951.389328:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389330:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204951.389332:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204951.389333:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204951.389337:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389344:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204951.389346:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389347:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204951.389349:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204951.389351:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204951.389354:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492352:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204951.389357:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204951.389360:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204951.389361:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389362:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204951.389365:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204951.389367:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204951.389371:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204951.389373:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204951.389376:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204951.389378:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543492352, portal 10 00000100:00000001:1.0:1788204951.389379:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204951.389380:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788204951.389385:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389390:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204951.389391:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204951.389393:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204951.389394:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543492352, offset 0 00000400:00000200:1.0:1788204951.389398:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204951.389402:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.389405:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204951.389408:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204951.389410:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204951.389412:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204951.389415:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204951.389417:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471195:471195:256:4294967295] 192.168.201.118@tcp LPNI seq info [471054:471054:8:4294967295] 00000400:00000200:1.0:1788204951.389422:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.389428:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204951.389433:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204951.389436:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204951.389440:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204951.389443:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204951.389446:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204951.389448:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788204951.389450:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204951.389451:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204951.389453:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204951.389455:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204951.389458:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204951.389459:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204951.389461:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204951.389463:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389468:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.389469:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389473:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389474:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204951.389477:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204951.389479:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788204951.389481:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204951.389483:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321310227 00000100:00000001:1.0:1788204951.389484:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321311493 00000100:00000001:1.0:1788204951.389486:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321310227 00000100:00000001:1.0:1788204951.389487:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321311493 00000100:00000001:1.0:1788204951.389488:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204951.389489:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389490:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204951.389492:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204951/real 1788204951] req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204951.389497:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204951.389498:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204951.389499:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204951.389500:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204951.389501:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204951.389502:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204951.389504:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204951.389509:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389512:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204951.389513:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389514:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.389515:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204951.389517:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389521:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204951.389523:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389526:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204951.389528:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389528:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.389529:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389530:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204951.389531:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204951.389533:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204951.389535:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204951.389536:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204951.389538:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204951.389542:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492352:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204951.389544:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204951.389546:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389550:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204951.389551:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389552:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204951.389553:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204951.389554:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204951.389555:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492480:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204951.389557:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204951.389558:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204951.389559:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389560:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204951.389562:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204951.389564:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204951.389565:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204951.389567:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204951.389569:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204951.389571:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543492480, portal 10 00000100:00000001:1.0:1788204951.389572:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204951.389573:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788204951.389576:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389580:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204951.389581:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204951.389583:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204951.389584:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543492480, offset 0 00000400:00000200:1.0:1788204951.389586:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204951.389588:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.389590:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204951.389592:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204951.389594:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204951.389595:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204951.389596:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204951.389598:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471196:471196:256:4294967295] 192.168.201.118@tcp LPNI seq info [471055:471055:8:4294967295] 00000400:00000200:1.0:1788204951.389601:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.389604:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204951.389607:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204951.389608:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204951.389610:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204951.389612:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204951.389614:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204951.389616:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788204951.389618:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204951.389631:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204951.389634:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204951.389636:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204951.389638:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204951.389639:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204951.389641:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204951.389643:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389648:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.389649:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389653:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389654:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204951.389656:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204951.389658:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204951.389660:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204951.389661:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321310228 00000100:00000001:1.0:1788204951.389663:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321311493 00000100:00000001:1.0:1788204951.389664:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321310228 00000100:00000001:1.0:1788204951.389666:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321311493 00000100:00000001:1.0:1788204951.389667:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204951.389668:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389670:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204951.389672:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204951/real 1788204951] req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204951.389676:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204951.389676:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204951.389677:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204951.389678:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204951.389679:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204951.389680:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204951.389682:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204951.389686:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389690:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204951.389691:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389692:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.389692:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204951.389694:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389698:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204951.389700:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389703:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204951.389704:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389705:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.389706:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389707:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204951.389709:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204951.389710:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204951.389712:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204951.389713:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204951.389716:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204951.389722:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492480:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204951.389725:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204951.389728:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389735:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204951.389736:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389738:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204951.389741:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204951.389743:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204951.389746:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492608:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204951.389748:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204951.389750:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204951.389751:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389753:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204951.389755:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204951.389757:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204951.389760:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204951.389763:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204951.389765:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204951.389768:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543492608, portal 4 00000100:00000001:1.0:1788204951.389769:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204951.389770:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788204951.389776:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389782:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204951.389784:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204951.389786:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204951.389788:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543492608, offset 0 00000400:00000200:1.0:1788204951.389792:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204951.389795:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.389798:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204951.389801:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204951.389803:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204951.389805:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204951.389807:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204951.389809:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471197:471197:256:4294967295] 192.168.201.118@tcp LPNI seq info [471056:471056:8:4294967295] 00000400:00000200:1.0:1788204951.389813:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.389818:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204951.389821:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204951.389824:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204951.389827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204951.389829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204951.389832:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204951.389834:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788204951.389837:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204951.389839:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204951.389841:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204951.389844:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204951.389847:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204951.389849:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204951.389850:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204951.389853:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389859:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.389862:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389871:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389873:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204951.389875:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204951.389877:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204951.389880:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204951.389882:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321310237 00000100:00000001:1.0:1788204951.389884:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321311493 00000100:00000001:1.0:1788204951.389886:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321310237 00000100:00000001:1.0:1788204951.389888:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321311493 00000100:00000001:1.0:1788204951.389890:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204951.389892:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.389894:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204951.389897:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204951/real 1788204951] req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204951.389903:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204951.389904:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204951.389905:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204951.389912:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204951.389914:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204951.389916:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204951.389918:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204951.389925:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389930:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204951.389931:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389932:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.389934:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204951.389937:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389943:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204951.389947:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389954:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204951.389956:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389957:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.389958:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.389960:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204951.389962:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204951.389964:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204951.389967:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204951.389969:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204951.389973:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204951.389981:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492608:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204951.389985:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204951.389988:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.389992:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204951.389993:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.389995:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204951.389996:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204951.389997:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204951.389999:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492736:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204951.390001:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204951.390002:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204951.390004:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204951.390005:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204951.390007:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204951.390008:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204951.390010:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204951.390012:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204951.390014:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204951.390016:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543492736, portal 4 00000100:00000001:1.0:1788204951.390017:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204951.390018:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788204951.390022:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390026:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204951.390028:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204951.390029:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204951.390030:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543492736, offset 0 00000400:00000200:1.0:1788204951.390032:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204951.390035:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.390037:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204951.390038:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204951.390040:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204951.390041:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204951.390043:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204951.390045:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471198:471198:256:4294967295] 192.168.201.118@tcp LPNI seq info [471057:471057:8:4294967295] 00000400:00000200:1.0:1788204951.390047:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204951.390051:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204951.390054:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204951.390055:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204951.390058:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204951.390060:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788204951.390065:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204951.390067:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788204951.390071:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204951.390073:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204951.390076:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204951.390079:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204951.390081:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204951.390083:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204951.390085:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204951.390088:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390113:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.390115:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204976 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390119:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.390120:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204951.390123:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204951.390125:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204951.390127:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204951.390129:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321310261 00000100:00000001:1.0:1788204951.390130:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321311493 00000100:00000001:1.0:1788204951.390132:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321310261 00000100:00000001:1.0:1788204951.390134:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321311493 00000100:00000001:1.0:1788204951.390136:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204951.390137:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.390139:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204951.390142:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204951/real 1788204951] req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204951.390150:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204951.390151:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204951.390153:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204951.390155:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204951.390156:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204951.390158:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204951.390162:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204951.390169:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390176:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204951.390177:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390179:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.390180:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204951.390183:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390189:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204951.390193:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390199:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204951.390201:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390203:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204951.390204:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390206:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204951.390208:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204951.390210:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204951.390213:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204951.390216:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204951.390219:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204951.390226:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492736:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204951.390230:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390232:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.390236:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543492352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390240:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204951.390241:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204951.390242:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204951.390244:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204951.390245:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204951.390246:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204951.390248:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204951.390250:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:1.0:1788204951.390252:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204951.390253:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204951.390255:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788204951.390257:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204951.390258:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390260:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.390263:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543492480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390269:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204951.390270:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204951.390271:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204951.390273:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204951.390275:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204951.390276:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204951.390277:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204951.390278:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788204951.390280:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204951.390281:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204951.390282:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788204951.390285:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204951.390286:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390287:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.390290:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543492608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390297:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204951.390298:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204951.390299:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204951.390302:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204951.390303:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204951.390304:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204951.390305:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204951.390306:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788204951.390308:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204951.390308:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204951.390309:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788204951.390311:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204951.390311:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390312:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204951.390314:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543492736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204976 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204951.390318:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204951.390319:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204951.390320:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204951.390321:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204951.390322:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204951.390323:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204951.390323:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204951.390324:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788204951.390325:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204951.390326:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204951.390326:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788204951.390327:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204951.390328:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390329:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204951.390331:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204951.390332:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204951.390333:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204951.390334:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204952.220221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204952.220221:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204952.220229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204952.220231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204952.220231:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204952.220239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204952.220239:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204952.220240:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204952.220253:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204952.220254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204952.220263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204952.220263:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204952.220264:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204952.220271:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204952.220274:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204952.220274:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204952.220281:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204952.220282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204952.220283:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204952.220291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204952.220292:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204952.220293:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204952.220298:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204952.220300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204952.220301:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204952.220302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204952.220375:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204952.220382:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204952.220386:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204952.732180:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321311493, at 4321311552 00000100:00000010:3.0:1788204952.732192:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00000100:00000040:2.0:1788204952.732206:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321311493, at 4321311552 00000100:00000010:2.0:1788204952.732218:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647ba0. 00010000:00000001:0.0:1788204953.244227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204953.244227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204953.244227:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204953.244240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204953.244242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204953.244242:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204953.244247:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204953.244249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204953.244254:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204953.244257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204953.244259:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204953.244261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204953.244266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204953.244266:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204953.244266:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204953.244272:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204953.244274:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204953.244280:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204953.244283:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204953.244284:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204953.244285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204953.244289:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204953.244292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204953.244295:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204953.244300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204953.244302:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204953.244359:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204953.244363:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204953.244367:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204954.268181:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204954.268188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204954.268188:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204954.268190:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204954.268195:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204954.268200:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204954.268202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204954.268202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204954.268203:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204954.268212:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204954.268213:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204954.268215:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204954.268222:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268223:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204954.268223:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204954.268231:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204954.268235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204954.268236:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268237:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788204954.268243:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204954.268244:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204954.268248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204954.268250:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204954.268251:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204954.268256:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204954.268258:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204954.268260:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204954.268267:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204954.268268:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204954.268269:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268271:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204954.268272:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268276:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204954.268283:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204954.268285:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204954.268287:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268288:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204954.268290:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268291:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204954.268293:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204954.268355:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204954.268360:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204954.268364:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204955.292192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204955.292192:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204955.292203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204955.292203:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204955.292209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204955.292209:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204955.292216:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204955.292217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204955.292219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204955.292223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204955.292223:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204955.292229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204955.292230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204955.292231:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204955.292245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204955.292246:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204955.292250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204955.292250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204955.292250:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204955.292261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204955.292262:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204955.292262:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204955.292280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204955.292283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204955.292287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204955.292289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204955.292360:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204955.292364:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204955.292367:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204956.316227:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204956.316232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204956.316235:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204956.316236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204956.316242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204956.316243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.316243:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.316245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204956.316255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204956.316257:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204956.316258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204956.316260:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204956.316263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.316266:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.316267:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204956.316270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204956.316272:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204956.316274:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204956.316275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.316275:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204956.316286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204956.316288:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204956.316292:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204956.316293:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.316293:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.316302:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204956.316384:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204956.316389:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204956.316393:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204956.508194:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204956.508205:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204956.508208:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204956.508211:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204956.508215:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204956.508222:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26642 00000400:00000200:3.0:1788204956.508232:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204956.508238:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204956.508242:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204956.508245:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508247:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204956.508249:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508251:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204956.508254:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204956.508255:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788204956.508257:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204956.508260:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788204956.508261:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788204956.508263:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788204956.508264:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508265:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204956.508266:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508267:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204956.508269:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788204956.508271:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508273:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204956.508275:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204956.508277:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204956.508279:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.508280:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204956.508282:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204956.508289:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788204956.508292:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788204956.508295:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204956.508296:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.508298:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204956.508301:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204956.508303:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204956.508309:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788204956.508314:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204956.508320:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204956.508327:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204956.508331:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204956.508334:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204956.508353:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204956.508356:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204956.508358:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204956.508359:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204956.508360:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204956.508362:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204956.508364:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26642 00000400:00000200:3.0:1788204956.508366:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204956.508368:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204956.508370:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204956.508372:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508373:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204956.508374:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508376:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204956.508377:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204956.508378:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788204956.508380:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204956.508382:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788204956.508383:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788204956.508384:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204956.508387:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508388:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204956.508390:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1788204956.508390:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508394:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204956.508395:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1788204956.508396:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:1.0:1788204956.508399:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788204956.508400:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508403:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204956.508405:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788204956.508407:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204956.508410:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204956.508415:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.508419:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204956.508422:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788204956.508423:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788204956.508425:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788204956.508428:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.508430:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788204956.508430:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.508433:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1788204956.508434:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 02000000:00000001:1.0:1788204956.508436:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788204956.508438:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00100000:1.0:1788204956.508441:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543492864:192.168.201.118@tcp:250:kworker.0 02000000:00000001:3.0:1788204956.508442:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204956.508445:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.508448:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788204956.508448:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204956.508451:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204956.508453:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204956.508453:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.508457:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.508460:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204956.508460:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 02000000:00000010:1.0:1788204956.508466:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000001:3.0:1788204956.508466:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.508469:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.508477:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:3.0:1788204956.508477:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1788204956.508481:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204956.508489:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788204956.508490:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204956.508494:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788204956.508496:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543492864, portal 25 00000100:00000040:3.0:1788204956.508500:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.508501:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204956.508510:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.508512:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00080000:3.0:1788204956.508513:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204956.508519:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788204956.508520:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204956.508522:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1788204956.508524:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204956.508525:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204956.508529:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26642 00000400:00000200:3.0:1788204956.508550:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204956.508553:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1788204956.508556:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204956.508560:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204956.508566:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:1.0:1788204956.508567:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508572:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1788204956.508573:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000040:3.0:1788204956.508576:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000200:1.0:1788204956.508580:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543492864, offset 0 00000100:00000001:3.0:1788204956.508581:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204956.508587:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204956.508588:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788204956.508591:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1788204956.508593:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000020:00000040:3.0:1788204956.508594:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788204956.508597:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788204956.508599:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788204956.508602:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788204956.508606:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204956.508606:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204956.508611:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508613:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204956.508615:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788204956.508618:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:1.0:1788204956.508619:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788204956.508620:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204956.508623:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204956.508627:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788204956.508629:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1788204956.508630:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.508632:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204956.508633:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204956.508635:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.508637:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204956.508641:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788204956.508644:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000200:1.0:1788204956.508645:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000001:3.0:1788204956.508648:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204956.508650:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.508653:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788204956.508653:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204956.508655:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204956.508658:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.508661:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 02000000:00000010:3.0:1788204956.508662:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:3.0:1788204956.508666:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.508668:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471199:471199:256:4294967295] 192.168.201.118@tcp LPNI seq info [471058:471058:8:4294967295] 00000100:00100000:3.0:1788204956.508673:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204956.508678:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204956.508685:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204956.508687:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204956.508693:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204956.508699:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788204956.508704:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204956.508708:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204956.508711:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:1.0:1788204956.508714:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508714:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204956.508716:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204956.508720:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204956.508722:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26642 00000800:00000010:1.0:1788204956.508723:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000400:00000200:3.0:1788204956.508727:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204956.508730:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204956.508733:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1788204956.508735:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:3.0:1788204956.508739:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.0:1788204956.508742:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000100:00000001:3.0:1788204956.508743:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204956.508748:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508752:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204956.508754:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204956.508756:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788204956.508757:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204956.508759:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788204956.508763:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:1.1:1788204956.508763:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:3.0:1788204956.508770:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:1.1:1788204956.508771:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000100:00000040:3.0:1788204956.508775:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508779:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1788204956.508781:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000040:3.0:1788204956.508785:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000010:1.0:1788204956.508788:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000100:00000001:3.0:1788204956.508788:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204956.508792:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:1.0:1788204956.508795:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1788204956.508795:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508800:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204956.508802:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788204956.508805:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1788204956.508805:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204956.508810:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.508812:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204956.508814:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000400:00000200:1.0:1788204956.508815:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000008:00000001:3.0:1788204956.508816:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.508819:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204956.508820:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000010:3.0:1788204956.508822:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788204956.508825:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788204956.508827:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1788204956.508827:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204956.508831:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.508832:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204956.508835:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1788204956.508836:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204956.508837:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204956.508842:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788204956.508846:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204956.508853:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204956.508861:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.508866:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.508884:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.508887:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204956.508890:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204956.508893:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.508894:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.508900:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788204956.508900:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.508903:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321316613 00000100:00000010:1.0:1788204956.508910:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16475a0. 00000100:00000001:3.0:1788204956.508911:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204956.508915:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788204956.508918:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00080000:3.0:1788204956.508918:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204956.508922:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204956.508923:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204956.508925:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204956.508927:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321316613 00000100:00080000:3.0:1788204956.508928:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26642 00000100:00000010:1.0:1788204956.508932:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16478a0. 00000400:00000200:3.0:1788204956.508936:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204956.508937:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000400:00000200:3.0:1788204956.508940:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788204956.508941:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788204956.508943:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204956.508945:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204956.508948:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204956.508949:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788204956.508952:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788204956.508957:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204956/real 1788204956] req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204956.508957:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508961:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204956.508966:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204956.508969:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204956.508972:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000001:1.0:1788204956.508974:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:3.0:1788204956.508974:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788204956.508976:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000001:1.0:1788204956.508978:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1788204956.508979:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000010:1.0:1788204956.508982:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000100:00000040:3.0:1788204956.508982:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508986:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788204956.508989:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00000040:3.0:1788204956.508990:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204956.508994:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000010:1.0:1788204956.508995:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000020:00000040:3.0:1788204956.508998:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:1.0:1788204956.509001:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1788204956.509002:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204956.509005:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204956.509007:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1788204956.509009:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204956.509010:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204956.509014:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.509018:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204956.509020:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788204956.509024:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.509028:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204956.509033:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000100:00000200:1.0:1788204956.509035:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788204956.509036:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204956.509040:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204956.509042:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204956.509046:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204956.509048:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204956.509050:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204956.509055:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788204956.509057:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204956.509061:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204956.509063:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204956.509066:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.509070:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204956.509071:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509074:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.509074:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204956.509080:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204956.509081:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509114:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204956.509116:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204956.509121:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26653) 00000100:00000040:1.0:1788204956.509123:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509132:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204956.509136:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509138:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509140:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509142:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204956.509144:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204956.509146:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204956.509150:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204956.509152:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204956.509156:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204956.509161:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543492864:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788204956.509165:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509166:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.509168:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543492864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509172:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204956.509174:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204956.509176:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204956.509179:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204956.509180:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204956.509181:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204956.509182:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204956.509184:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788204956.509187:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204956.509188:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204956.509189:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788204956.509191:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204956.509192:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509194:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509199:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204956.509200:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509201:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204956.509202:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204956.509203:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204956.509205:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509210:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204956.509211:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.509212:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204956.509213:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204956.509214:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204956.509216:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492992:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204956.509218:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204956.509219:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204956.509220:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.509221:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204956.509223:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204956.509225:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.509227:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204956.509229:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204956.509231:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204956.509234:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543492992, portal 10 00000100:00000001:1.0:1788204956.509235:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204956.509236:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788204956.509240:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509244:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204956.509246:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204956.509249:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204956.509251:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543492992, offset 0 00000400:00000200:1.0:1788204956.509255:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204956.509258:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204956.509262:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204956.509264:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204956.509267:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204956.509269:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204956.509272:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204956.509275:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471200:471200:256:4294967295] 192.168.201.118@tcp LPNI seq info [471059:471059:8:4294967295] 00000400:00000200:1.0:1788204956.509279:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204956.509285:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204956.509290:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204956.509293:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204956.509297:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204956.509300:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1788204956.509302:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204956.509304:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1788204956.509314:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204956.509315:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204956.509317:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204956.509319:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204956.509321:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204956.509323:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204956.509324:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204956.509327:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509332:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.509334:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509337:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509338:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204956.509341:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204956.509343:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788204956.509345:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204956.509347:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321315347 00000100:00000001:1.0:1788204956.509349:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321316613 00000100:00000001:1.0:1788204956.509350:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321315347 00000100:00000001:1.0:1788204956.509352:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321316613 00000100:00000001:1.0:1788204956.509353:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204956.509354:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509356:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204956.509358:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204956/real 1788204956] req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204956.509362:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204956.509363:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204956.509364:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204956.509365:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204956.509366:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204956.509367:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204956.509369:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204956.509374:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509377:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204956.509378:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509379:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509380:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204956.509381:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509385:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204956.509387:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509391:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204956.509392:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509393:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509393:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509394:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204956.509396:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204956.509397:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204956.509399:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204956.509401:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204956.509408:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204956.509413:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543492992:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204956.509415:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204956.509419:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509423:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204956.509424:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.509425:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204956.509427:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204956.509428:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204956.509429:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493120:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204956.509431:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204956.509433:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204956.509433:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.509434:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204956.509436:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204956.509437:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.509439:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204956.509441:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204956.509443:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204956.509444:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543493120, portal 10 00000100:00000001:1.0:1788204956.509446:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204956.509447:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788204956.509450:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509457:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204956.509459:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204956.509460:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204956.509461:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543493120, offset 0 00000400:00000200:1.0:1788204956.509463:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204956.509465:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204956.509467:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204956.509469:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204956.509471:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204956.509472:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204956.509473:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204956.509476:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471201:471201:256:4294967295] 192.168.201.118@tcp LPNI seq info [471060:471060:8:4294967295] 00000400:00000200:1.0:1788204956.509478:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204956.509482:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204956.509485:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204956.509487:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204956.509489:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204956.509491:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1788204956.509493:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204956.509494:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1788204956.509497:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204956.509498:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204956.509500:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204956.509503:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204956.509506:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204956.509508:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204956.509509:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204956.509512:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509519:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.509522:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509544:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509546:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204956.509549:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204956.509552:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204956.509555:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204956.509558:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321315348 00000100:00000001:1.0:1788204956.509560:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321316613 00000100:00000001:1.0:1788204956.509562:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321315348 00000100:00000001:1.0:1788204956.509564:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321316613 00000100:00000001:1.0:1788204956.509566:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204956.509567:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509569:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204956.509573:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204956/real 1788204956] req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204956.509581:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204956.509582:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204956.509583:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204956.509586:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204956.509587:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204956.509590:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204956.509593:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204956.509601:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509605:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204956.509606:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509607:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509608:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204956.509610:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509614:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204956.509617:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509621:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204956.509622:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509623:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509623:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509625:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204956.509626:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204956.509627:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204956.509629:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204956.509630:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204956.509632:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204956.509636:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493120:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204956.509639:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204956.509641:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509644:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204956.509645:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.509647:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204956.509648:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204956.509649:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204956.509650:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493248:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204956.509652:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204956.509653:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204956.509654:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.509655:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204956.509658:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204956.509659:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.509661:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204956.509663:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204956.509665:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204956.509667:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543493248, portal 4 00000100:00000001:1.0:1788204956.509668:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204956.509669:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788204956.509672:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509676:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204956.509677:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204956.509678:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204956.509680:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543493248, offset 0 00000400:00000200:1.0:1788204956.509684:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204956.509688:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204956.509691:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204956.509694:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204956.509696:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204956.509699:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204956.509701:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204956.509704:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471202:471202:256:4294967295] 192.168.201.118@tcp LPNI seq info [471061:471061:8:4294967295] 00000400:00000200:1.0:1788204956.509708:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204956.509714:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204956.509718:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204956.509721:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204956.509725:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204956.509728:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1788204956.509732:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204956.509734:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1788204956.509738:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204956.509740:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204956.509743:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204956.509746:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204956.509749:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204956.509751:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204956.509753:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204956.509756:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509763:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.509766:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509772:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509774:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204956.509777:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204956.509780:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204956.509783:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204956.509785:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321315357 00000100:00000001:1.0:1788204956.509787:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321316613 00000100:00000001:1.0:1788204956.509788:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321315357 00000100:00000001:1.0:1788204956.509789:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321316613 00000100:00000001:1.0:1788204956.509790:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204956.509792:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509793:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204956.509795:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204956/real 1788204956] req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204956.509800:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204956.509801:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204956.509802:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204956.509804:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204956.509805:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204956.509807:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204956.509808:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204956.509813:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509817:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204956.509817:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509818:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509819:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204956.509821:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509825:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204956.509827:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509830:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204956.509832:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509832:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509833:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509834:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204956.509835:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204956.509837:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204956.509838:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204956.509839:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204956.509841:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204956.509845:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493248:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204956.509847:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204956.509849:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509853:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204956.509854:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.509855:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204956.509856:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204956.509857:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204956.509859:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493376:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204956.509861:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204956.509862:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204956.509863:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204956.509864:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204956.509866:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204956.509867:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204956.509869:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204956.509870:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204956.509872:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204956.509874:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543493376, portal 4 00000100:00000001:1.0:1788204956.509875:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204956.509876:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788204956.509879:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509883:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204956.509884:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204956.509886:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204956.509887:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543493376, offset 0 00000400:00000200:1.0:1788204956.509889:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204956.509891:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204956.509893:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204956.509895:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204956.509897:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204956.509898:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204956.509900:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204956.509902:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471203:471203:256:4294967295] 192.168.201.118@tcp LPNI seq info [471062:471062:8:4294967295] 00000400:00000200:1.0:1788204956.509904:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204956.509908:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204956.509911:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204956.509912:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204956.509915:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204956.509916:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000200:1.1:1788204956.509919:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204956.509921:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 11 secs later) 00000800:00000100:1.0:1788204956.509923:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204956.509925:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204956.509926:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204956.509928:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204956.509930:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204956.509931:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204956.509932:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204956.509934:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509938:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.509940:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204981 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509943:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509944:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204956.509946:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204956.509948:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204956.509949:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204956.509950:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321315380 00000100:00000001:1.0:1788204956.509952:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321316613 00000100:00000001:1.0:1788204956.509953:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321315380 00000100:00000001:1.0:1788204956.509955:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321316613 00000100:00000001:1.0:1788204956.509956:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204956.509957:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.509958:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204956.509960:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204956/real 1788204956] req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204956.509964:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204956.509965:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204956.509966:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204956.509967:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204956.509968:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204956.509969:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204956.509971:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204956.509975:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509979:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204956.509979:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509980:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509981:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204956.509983:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509986:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204956.509988:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.509991:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204956.509992:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509993:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204956.509994:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.509995:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204956.509996:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204956.509997:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204956.509999:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204956.510000:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204956.510002:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204956.510006:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493376:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204956.510008:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.510009:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.510011:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543492992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.510014:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204956.510015:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204956.510016:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204956.510017:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204956.510018:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204956.510019:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204956.510020:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204956.510021:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788204956.510022:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204956.510023:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204956.510023:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788204956.510025:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204956.510026:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.510027:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.510028:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543493120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.510032:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204956.510033:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204956.510033:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204956.510034:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204956.510035:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204956.510036:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204956.510037:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204956.510038:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788204956.510039:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204956.510040:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204956.510040:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788204956.510042:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204956.510043:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.510043:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.510045:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543493248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.510048:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204956.510049:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204956.510049:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204956.510051:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204956.510051:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204956.510052:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204956.510053:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204956.510054:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788204956.510055:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204956.510055:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204956.510056:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788204956.510058:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204956.510058:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.510059:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204956.510061:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543493376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204981 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204956.510064:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204956.510064:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204956.510065:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204956.510066:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204956.510067:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204956.510068:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204956.510068:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204956.510069:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788204956.510070:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204956.510071:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204956.510072:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788204956.510073:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204956.510074:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.510075:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204956.510077:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204956.510078:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204956.510079:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204956.510081:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204957.340205:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204957.340210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204957.340214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204957.340215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204957.340218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204957.340223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204957.340224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204957.340225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204957.340231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204957.340236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204957.340236:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204957.340237:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204957.340240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204957.340246:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204957.340247:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204957.340250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204957.340252:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204957.340252:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204957.340253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204957.340264:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204957.340265:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204957.340272:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204957.340277:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204957.340281:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204957.340284:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204957.340289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204957.340337:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204957.340341:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204957.340344:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204957.852251:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321316613, at 4321316672 00000100:00000040:2.0:1788204957.852253:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321316613, at 4321316672 00000100:00000010:3.0:1788204957.852271:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16478a0. 00000100:00000010:2.0:1788204957.852276:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00000100:00000001:1.0:1788204958.364219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204958.364219:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204958.364229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204958.364229:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204958.364233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204958.364233:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204958.364239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204958.364241:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204958.364242:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204958.364251:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204958.364252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204958.364252:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204958.364259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204958.364259:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204958.364263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204958.364265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204958.364266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204958.364267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204958.364269:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204958.364275:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204958.364278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204958.364278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204958.364280:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204958.364285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204958.364294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204958.364296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204958.364363:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204958.364369:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204958.364374:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204959.388196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204959.388196:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204959.388202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204959.388207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204959.388207:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204959.388208:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204959.388211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204959.388213:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204959.388213:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204959.388217:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204959.388221:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:1.0:1788204959.388222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204959.388223:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204959.388224:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204959.388230:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.388230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204959.388231:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204959.388233:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.388236:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.388237:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204959.388240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204959.388241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204959.388243:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204959.388244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204959.388246:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204959.388248:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204959.388251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204959.388252:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204959.388253:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204959.388261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204959.388269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204959.388271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204959.388344:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204959.388349:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204959.388352:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204960.412207:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204960.412207:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204960.412216:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204960.412218:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204960.412218:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204960.412220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204960.412221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204960.412224:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204960.412226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204960.412228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204960.412229:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204960.412233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204960.412233:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204960.412234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204960.412237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204960.412238:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204960.412239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204960.412242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204960.412242:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204960.412244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204960.412245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204960.412247:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204960.412249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204960.412251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204960.412254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204960.412255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204960.412319:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204960.412323:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204960.412326:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204961.436176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204961.436176:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204961.436185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204961.436186:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204961.436188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.436189:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204961.436190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204961.436195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204961.436197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204961.436197:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204961.436201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204961.436202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.436203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204961.436203:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204961.436206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204961.436207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204961.436208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.436208:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.436215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204961.436215:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204961.436216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204961.436219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.436219:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.436221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204961.436222:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.436225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204961.436292:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204961.436297:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204961.436305:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204961.628219:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204961.628230:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204961.628232:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204961.628236:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204961.628240:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204961.628247:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26648 00000400:00000200:3.0:1788204961.628254:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204961.628259:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204961.628262:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204961.628264:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628267:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204961.628269:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628272:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204961.628275:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204961.628276:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788204961.628278:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204961.628281:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788204961.628283:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788204961.628285:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788204961.628286:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628287:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204961.628288:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628289:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204961.628292:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788204961.628295:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628296:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204961.628298:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204961.628302:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204961.628303:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.628305:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204961.628307:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204961.628315:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788204961.628318:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788204961.628322:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204961.628323:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.628325:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204961.628327:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204961.628330:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204961.628336:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412000. 00000100:00000001:3.0:1788204961.628341:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204961.628348:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204961.628356:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204961.628359:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204961.628362:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204961.628381:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204961.628384:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204961.628386:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204961.628387:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204961.628388:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204961.628390:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204961.628392:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26648 00000400:00000200:3.0:1788204961.628395:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204961.628397:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204961.628399:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204961.628402:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628404:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204961.628406:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628408:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204961.628410:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204961.628411:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788204961.628412:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204961.628414:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788204961.628416:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788204961.628417:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204961.628436:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628439:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204961.628442:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628462:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204961.628464:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:1.0:1788204961.628467:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788204961.628467:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628472:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204961.628475:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788204961.628478:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788204961.628480:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204961.628484:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.628486:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788204961.628487:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204961.628495:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204961.628498:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.628504:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204961.628506:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788204961.628507:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788204961.628513:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204961.628518:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204961.628522:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.628528:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204961.628534:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204961.628536:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204961.628544:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411c00. 00000100:00000001:1.0:1788204961.628545:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204961.628549:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.628550:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.628559:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1788204961.628560:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788204961.628565:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204961.628569:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788204961.628573:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.628574:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204961.628587:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543493504:192.168.201.118@tcp:250:kworker.0 00000100:00000040:3.0:1788204961.628588:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.628607:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788204961.628609:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204961.628612:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204961.628614:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1788204961.628615:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788204961.628616:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788204961.628620:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204961.628621:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788204961.628628:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788204961.628628:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204961.628634:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26648 00000400:00000200:3.0:1788204961.628637:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204961.628640:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 02000000:00000010:1.0:1788204961.628641:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 00000100:00080000:3.0:1788204961.628643:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788204961.628646:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204961.628652:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628655:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204961.628658:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788204961.628662:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204961.628662:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204961.628670:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204961.628671:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000010:1.0:1788204961.628674:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204961.628686:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000020:00000001:3.0:1788204961.628691:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204961.628694:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000200:1.0:1788204961.628695:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543493504, portal 25 00000020:00000001:3.0:1788204961.628697:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788204961.628706:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788204961.628707:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1788204961.628713:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204961.628714:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000001:3.0:1788204961.628716:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204961.628721:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628723:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788204961.628730:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788204961.628732:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788204961.628741:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628749:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204961.628755:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204961.628759:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204961.628762:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.628764:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204961.628766:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204961.628767:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.628768:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.628769:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:3.0:1788204961.628771:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788204961.628774:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204961.628779:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1788204961.628780:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788204961.628782:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204961.628788:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000001:3.0:1788204961.628788:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204961.628792:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204961.628797:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788204961.628798:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543493504, offset 0 02000000:00000010:3.0:1788204961.628803:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412c00. 00000400:00000200:1.0:1788204961.628812:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628814:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204961.628822:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204961.628830:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204961.628842:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788204961.628848:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788204961.628848:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204961.628862:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1788204961.628862:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204961.628871:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204961.628875:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788204961.628884:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204961.628891:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00080000:3.0:1788204961.628893:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204961.628901:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:1.0:1788204961.628902:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471204:471204:256:4294967295] 192.168.201.118@tcp LPNI seq info [471063:471063:8:4294967295] 00000100:00000001:3.0:1788204961.628904:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204961.628908:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788204961.628913:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204961.628914:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204961.628921:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26648 00000400:00000200:1.0:1788204961.628932:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204961.628941:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204961.628947:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000010:1.0:1788204961.628952:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000400:00000200:3.0:1788204961.628956:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204961.628960:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1788204961.628968:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:3.0:1788204961.628969:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.0:1788204961.628976:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000100:00000001:3.0:1788204961.628976:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204961.628983:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.628991:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.1:1788204961.628991:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:3.0:1788204961.628999:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:1.1:1788204961.628999:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000020:00000040:3.0:1788204961.629003:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000800:00000100:1.0:1788204961.629009:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000001:3.0:1788204961.629009:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204961.629013:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000800:00000010:1.0:1788204961.629015:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000020:00000001:3.0:1788204961.629018:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:1.0:1788204961.629027:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1788204961.629031:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788204961.629037:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.629039:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204961.629040:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.629041:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204961.629044:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:1.0:1788204961.629046:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1788204961.629051:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204961.629056:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204961.629057:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788204961.629060:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 02000000:00000001:3.0:1788204961.629060:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204961.629068:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204961.629069:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:3.0:1788204961.629070:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204961.629074:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788204961.629078:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.629080:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1788204961.629080:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204961.629087:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788204961.629126:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204961.629128:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1788204961.629130:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204961.629131:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.629137:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204961.629140:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204961.629143:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204961.629152:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788204961.629159:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204961.629171:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204961.629188:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788204961.629189:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204961.629194:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204961.629198:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204961.629203:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629219:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.629223:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204961.629231:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.629235:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204961.629235:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204961.629242:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204961.629244:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204961.629246:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788204961.629254:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204961.629255:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204961.629264:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321321734 00000100:00080000:3.0:1788204961.629265:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26648 00000400:00000200:3.0:1788204961.629276:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:1.0:1788204961.629278:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647240. 00000400:00000200:3.0:1788204961.629279:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204961.629286:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204961.629290:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.629292:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204961.629293:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000040:3.0:1788204961.629294:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.629300:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204961.629302:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204961.629303:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204961.629304:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204961.629306:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788204961.629307:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321321734 00000020:00000001:3.0:1788204961.629308:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788204961.629315:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000010:1.0:1788204961.629318:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647c00. 00000100:00000040:3.0:1788204961.629318:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.629328:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204961.629330:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000040:3.0:1788204961.629331:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204961.629338:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204961.629341:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1788204961.629342:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788204961.629351:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204961.629352:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.629354:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204961.629359:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788204961.629360:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788204961.629362:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204961.629365:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.629366:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204961.629368:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788204961.629373:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204961/real 1788204961] req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788204961.629373:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.629374:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204961.629381:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788204961.629385:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204961.629396:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204961.629400:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204961.629404:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1788204961.629406:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788204961.629406:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1788204961.629410:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788204961.629414:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204961.629415:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 02000000:00000010:3.0:1788204961.629422:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000400:00000200:1.0:1788204961.629423:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204961.629429:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788204961.629432:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.629434:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1788204961.629456:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788204961.629458:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204961.629481:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204961.629485:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788204961.629491:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204961.629496:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629520:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204961.629524:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.629525:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629530:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1788204961.629532:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26658) 00000100:00000001:1.0:1788204961.629533:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204961.629544:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629553:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204961.629559:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629566:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204961.629570:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629571:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.629573:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629576:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204961.629578:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204961.629581:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204961.629586:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204961.629588:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204961.629593:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204961.629600:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543493504:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788204961.629605:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629607:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.629611:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543493504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629617:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204961.629620:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204961.629624:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204961.629628:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204961.629630:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204961.629634:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204961.629636:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204961.629638:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788204961.629643:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204961.629646:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204961.629647:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788204961.629651:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204961.629652:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629654:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629663:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204961.629665:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.629669:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204961.629670:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204961.629672:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204961.629675:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629683:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204961.629684:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.629687:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204961.629689:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204961.629691:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204961.629694:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493632:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204961.629697:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204961.629700:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204961.629701:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.629703:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204961.629706:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204961.629708:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204961.629712:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204961.629715:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204961.629718:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204961.629721:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543493632, portal 10 00000100:00000001:1.0:1788204961.629723:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204961.629725:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788204961.629731:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629738:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204961.629741:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204961.629743:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204961.629745:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543493632, offset 0 00000400:00000200:1.0:1788204961.629749:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204961.629753:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204961.629757:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204961.629761:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204961.629763:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204961.629765:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204961.629767:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204961.629771:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471205:471205:256:4294967295] 192.168.201.118@tcp LPNI seq info [471064:471064:8:4294967295] 00000400:00000200:1.0:1788204961.629775:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204961.629782:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204961.629787:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204961.629790:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204961.629793:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204961.629796:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788204961.629801:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204961.629804:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1788204961.629808:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204961.629809:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204961.629812:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204961.629816:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204961.629818:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204961.629820:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204961.629822:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204961.629826:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629833:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.629836:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629847:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.629848:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204961.629852:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204961.629856:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788204961.629859:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204961.629861:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321320467 00000100:00000001:1.0:1788204961.629863:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321321734 00000100:00000001:1.0:1788204961.629866:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321320467 00000100:00000001:1.0:1788204961.629868:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321321734 00000100:00000001:1.0:1788204961.629871:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204961.629872:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.629874:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204961.629878:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204961/real 1788204961] req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204961.629895:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204961.629897:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204961.629899:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204961.629916:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204961.629918:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204961.629920:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204961.629924:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204961.629931:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629938:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204961.629939:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629941:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.629943:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204961.629946:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629953:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204961.629956:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.629962:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204961.629965:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629966:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.629968:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.629969:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204961.629972:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204961.629974:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204961.629978:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204961.629980:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204961.629984:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204961.629991:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493632:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204961.629996:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204961.630000:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630006:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204961.630008:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.630010:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204961.630011:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204961.630013:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204961.630017:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493760:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204961.630020:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204961.630022:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204961.630024:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.630026:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204961.630029:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204961.630031:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204961.630034:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204961.630037:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204961.630040:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204961.630043:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543493760, portal 10 00000100:00000001:1.0:1788204961.630045:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204961.630047:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788204961.630052:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630059:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204961.630061:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204961.630063:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204961.630065:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543493760, offset 0 00000400:00000200:1.0:1788204961.630069:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204961.630073:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204961.630076:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204961.630079:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204961.630082:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204961.630084:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204961.630086:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204961.630113:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471206:471206:256:4294967295] 192.168.201.118@tcp LPNI seq info [471065:471065:8:4294967295] 00000400:00000200:1.0:1788204961.630117:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204961.630123:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204961.630128:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204961.630131:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204961.630135:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204961.630137:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788204961.630141:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204961.630143:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1788204961.630147:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204961.630149:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204961.630153:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204961.630156:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204961.630159:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204961.630160:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204961.630163:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204961.630166:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630173:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.630176:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630182:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.630184:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204961.630187:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204961.630190:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204961.630193:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204961.630195:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321320469 00000100:00000001:1.0:1788204961.630197:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321321734 00000100:00000001:1.0:1788204961.630199:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321320469 00000100:00000001:1.0:1788204961.630201:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321321734 00000100:00000001:1.0:1788204961.630203:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204961.630205:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.630207:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204961.630211:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204961/real 1788204961] req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204961.630218:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204961.630219:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204961.630221:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204961.630223:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204961.630225:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204961.630227:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204961.630230:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204961.630238:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630245:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204961.630246:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630248:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.630249:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204961.630253:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630260:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204961.630263:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630269:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204961.630272:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630274:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.630275:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630277:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204961.630280:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204961.630283:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204961.630287:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204961.630289:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204961.630292:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204961.630300:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493760:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204961.630304:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204961.630308:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630315:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204961.630316:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.630319:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204961.630320:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204961.630322:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204961.630325:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493888:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204961.630328:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204961.630331:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204961.630332:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.630334:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204961.630337:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204961.630339:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204961.630343:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204961.630345:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204961.630348:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204961.630351:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543493888, portal 4 00000100:00000001:1.0:1788204961.630353:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204961.630355:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788204961.630360:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630367:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204961.630369:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204961.630372:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204961.630374:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543493888, offset 0 00000400:00000200:1.0:1788204961.630378:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204961.630381:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204961.630384:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204961.630387:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204961.630389:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204961.630391:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204961.630394:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204961.630397:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471207:471207:256:4294967295] 192.168.201.118@tcp LPNI seq info [471066:471066:8:4294967295] 00000400:00000200:1.0:1788204961.630401:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204961.630406:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204961.630410:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204961.630414:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204961.630417:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204961.630420:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788204961.630424:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204961.630427:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1788204961.630430:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204961.630432:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204961.630435:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204961.630438:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204961.630441:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204961.630455:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204961.630458:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204961.630462:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630469:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.630472:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630478:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.630480:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204961.630483:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204961.630486:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204961.630489:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204961.630491:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321320478 00000100:00000001:1.0:1788204961.630494:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321321734 00000100:00000001:1.0:1788204961.630496:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321320478 00000100:00000001:1.0:1788204961.630498:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321321734 00000100:00000001:1.0:1788204961.630500:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204961.630503:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.630505:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204961.630508:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204961/real 1788204961] req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204961.630516:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204961.630517:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204961.630520:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204961.630522:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204961.630524:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204961.630526:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204961.630529:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204961.630537:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630544:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204961.630545:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630547:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.630548:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204961.630552:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630559:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204961.630562:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630569:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204961.630571:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630572:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.630574:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630576:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204961.630578:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204961.630580:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204961.630584:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204961.630586:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204961.630590:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204961.630597:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543493888:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204961.630601:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204961.630604:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630611:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204961.630612:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.630615:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204961.630616:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204961.630618:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204961.630621:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494016:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204961.630625:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204961.630626:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204961.630628:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204961.630630:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204961.630633:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204961.630635:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204961.630638:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204961.630641:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204961.630644:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204961.630647:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543494016, portal 4 00000100:00000001:1.0:1788204961.630649:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204961.630650:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788204961.630656:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630662:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204961.630665:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204961.630667:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204961.630669:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543494016, offset 0 00000400:00000200:1.0:1788204961.630673:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204961.630677:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204961.630680:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204961.630683:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204961.630685:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204961.630688:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204961.630690:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204961.630693:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471208:471208:256:4294967295] 192.168.201.118@tcp LPNI seq info [471067:471067:8:4294967295] 00000400:00000200:1.0:1788204961.630697:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204961.630703:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204961.630707:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204961.630710:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204961.630714:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204961.630716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1788204961.630720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204961.630722:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1788204961.630726:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204961.630728:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204961.630730:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204961.630734:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204961.630736:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204961.630738:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204961.630740:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204961.630743:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630751:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.630754:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204986 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630760:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.630761:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204961.630765:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204961.630768:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204961.630771:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204961.630773:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321320501 00000100:00000001:1.0:1788204961.630775:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321321734 00000100:00000001:1.0:1788204961.630777:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321320501 00000100:00000001:1.0:1788204961.630779:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321321734 00000100:00000001:1.0:1788204961.630781:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204961.630783:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.630785:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204961.630789:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204961/real 1788204961] req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204961.630796:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204961.630798:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204961.630800:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204961.630802:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204961.630804:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204961.630806:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204961.630808:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204961.630816:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630822:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204961.630824:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630825:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.630826:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204961.630830:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630836:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204961.630839:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630846:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204961.630849:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630850:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204961.630852:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630853:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204961.630855:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204961.630858:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204961.630861:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204961.630863:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204961.630868:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204961.630875:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494016:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204961.630879:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630881:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.630883:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543493632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630890:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204961.630892:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204961.630894:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204961.630897:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204961.630899:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204961.630900:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204961.630902:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204961.630905:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788204961.630908:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204961.630910:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204961.630912:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788204961.630915:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204961.630916:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630918:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.630921:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543493760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630928:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204961.630929:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204961.630931:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204961.630933:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204961.630934:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204961.630935:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204961.630937:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204961.630939:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:1.0:1788204961.630941:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204961.630942:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204961.630944:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788204961.630946:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204961.630947:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630949:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.630952:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543493888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630957:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204961.630958:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204961.630960:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204961.630962:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204961.630963:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204961.630965:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204961.630966:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204961.630968:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788204961.630970:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204961.630971:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204961.630973:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788204961.630976:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204961.630978:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.630979:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204961.630982:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543494016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204986 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204961.630993:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204961.630994:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204961.630996:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204961.630998:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204961.630999:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204961.631001:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204961.631002:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204961.631004:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788204961.631006:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204961.631007:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204961.631008:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788204961.631010:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204961.631011:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.631013:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204961.631019:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204961.631020:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204961.631023:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204961.631025:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204962.460194:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204962.460197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204962.460202:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204962.460206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204962.460209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204962.460209:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204962.460212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204962.460214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204962.460218:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204962.460219:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204962.460224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204962.460226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204962.460227:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204962.460229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204962.460231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204962.460233:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204962.460236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204962.460237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204962.460242:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204962.460243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204962.460243:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204962.460249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204962.460251:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204962.460252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204962.460256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204962.460258:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204962.460306:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204962.460309:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204962.460312:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204962.972198:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321321734, at 4321321792 00000100:00000040:3.0:1788204962.972198:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321321734, at 4321321792 00000100:00000010:2.0:1788204962.972214:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00000100:00000010:3.0:1788204962.972216:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c00. 00000100:00000001:1.0:1788204963.484182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204963.484182:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204963.484191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204963.484191:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204963.484194:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204963.484195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204963.484203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204963.484203:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204963.484208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204963.484208:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204963.484213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204963.484213:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204963.484219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204963.484219:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204963.484223:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204963.484224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204963.484227:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204963.484228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.484951:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204963.484954:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.484960:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204963.484963:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204963.484965:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204963.484968:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.484972:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204963.484974:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.484976:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.484978:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.484981:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204963.484983:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204963.484984:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204963.484986:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.484987:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204963.484989:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.484991:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.485001:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.485005:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204963.485006:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204963.485008:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204963.485009:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.485011:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204963.485012:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.485014:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204963.485016:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204963.485082:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204963.485086:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204963.485114:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204964.508184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204964.508184:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204964.508190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204964.508191:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204964.508195:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204964.508196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204964.508197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204964.508202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204964.508202:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204964.508204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204964.508206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204964.508208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204964.508209:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204964.508210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204964.508214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204964.508214:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204964.508216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204964.508220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204964.508221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204964.508222:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204964.508223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204964.508224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204964.508226:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204964.508227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204964.508227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204964.508233:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204964.508281:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204964.508285:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204964.508289:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204965.532211:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204965.532211:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204965.532215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204965.532221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.532221:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204965.532222:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204965.532227:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.532227:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204965.532237:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204965.532237:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204965.532239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204965.532243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204965.532245:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204965.532249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204965.532249:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204965.532251:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204965.532253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.532253:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204965.532256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204965.532260:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204965.532261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204965.532262:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204965.532265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.532267:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204965.532269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204965.532273:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204965.532353:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204965.532359:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204965.532362:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204966.556185:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204966.556186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204966.556189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204966.556193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.556193:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204966.556194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204966.556195:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.556198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204966.556200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204966.556200:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204966.556203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204966.556204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.556204:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204966.556207:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.556210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204966.556211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204966.556211:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204966.556213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.556214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204966.556215:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204966.556217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204966.556219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.556220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204966.556221:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.556224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.556228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204966.556276:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204966.556281:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204966.556284:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788204966.748196:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204966.748207:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788204966.748208:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204966.748211:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204966.748214:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204966.748219:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26653 00000400:00000200:3.0:1788204966.748227:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204966.748232:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204966.748234:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204966.748236:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748238:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204966.748240:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748241:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204966.748244:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204966.748245:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788204966.748247:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204966.748250:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788204966.748251:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788204966.748252:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788204966.748255:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748256:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204966.748257:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748259:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204966.748260:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788204966.748263:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748264:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204966.748266:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204966.748268:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204966.748270:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748271:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204966.748273:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204966.748283:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788204966.748286:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788204966.748290:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204966.748292:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748294:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204966.748297:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204966.748301:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204966.748314:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:3.0:1788204966.748320:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204966.748328:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204966.748337:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204966.748341:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204966.748346:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204966.748390:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204966.748392:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204966.748395:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788204966.748396:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204966.748397:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204966.748399:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204966.748401:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26653 00000400:00000200:3.0:1788204966.748404:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204966.748406:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204966.748407:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204966.748409:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748410:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204966.748412:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748413:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204966.748415:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204966.748416:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788204966.748417:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204966.748418:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788204966.748419:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788204966.748421:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204966.748422:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748424:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204966.748425:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748426:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204966.748428:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788204966.748430:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748432:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.748434:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788204966.748434:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788204966.748440:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788204966.748440:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204966.748443:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788204966.748444:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748448:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:1.0:1788204966.748453:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788204966.748454:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204966.748461:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748466:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.748472:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:3.0:1788204966.748473:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788204966.748476:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788204966.748477:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788204966.748482:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204966.748482:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204966.748486:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.748487:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204966.748488:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204966.748492:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204966.748493:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204966.748495:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204966.748500:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543494144:192.168.201.118@tcp:250:kworker.0 02000000:00000010:3.0:1788204966.748501:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:1.0:1788204966.748507:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788204966.748508:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.748513:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1788204966.748516:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204966.748523:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204966.748526:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.748527:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204966.748530:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204966.748536:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788204966.748542:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204966.748545:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 00000100:00080000:3.0:1788204966.748545:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788204966.748550:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204966.748551:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788204966.748554:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204966.748556:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788204966.748560:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204966.748560:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204966.748565:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26653 00000400:00000010:1.0:1788204966.748566:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:3.0:1788204966.748571:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204966.748575:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000010:1.0:1788204966.748576:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00080000:3.0:1788204966.748578:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204966.748582:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000200:1.0:1788204966.748584:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543494144, portal 25 00000100:00000001:3.0:1788204966.748585:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204966.748589:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1788204966.748590:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204966.748593:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000001:3.0:1788204966.748594:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204966.748600:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204966.748601:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788204966.748602:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204966.748605:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000040:1.0:1788204966.748606:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788204966.748607:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788204966.748611:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788204966.748616:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748620:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204966.748623:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204966.748628:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788204966.748628:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204966.748629:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000040:1.0:1788204966.748634:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788204966.748635:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748639:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204966.748640:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 02000000:00000001:3.0:1788204966.748640:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204966.748644:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1788204966.748646:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543494144, offset 0 02000000:00000001:3.0:1788204966.748647:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748651:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204966.748653:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788204966.748655:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748657:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204966.748660:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000010:3.0:1788204966.748662:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788204966.748664:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204966.748666:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204966.748667:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748668:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204966.748669:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1788204966.748671:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788204966.748672:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204966.748677:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788204966.748680:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204966.748681:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00100000:3.0:1788204966.748686:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204966.748690:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788204966.748692:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204966.748693:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204966.748696:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204966.748701:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204966.748704:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788204966.748709:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204966.748713:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1788204966.748719:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:1.0:1788204966.748720:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748723:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788204966.748726:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788204966.748729:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471209:471209:256:4294967295] 192.168.201.118@tcp LPNI seq info [471068:471068:8:4294967295] 00000100:00000001:3.0:1788204966.748731:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204966.748737:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26653 00000400:00000200:1.0:1788204966.748740:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204966.748745:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204966.748749:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204966.748752:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204966.748757:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748761:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788204966.748765:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000040:3.0:1788204966.748766:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748770:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204966.748774:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204966.748775:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788204966.748776:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204966.748779:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000800:00000200:1.0:1788204966.748780:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000020:00000001:3.0:1788204966.748781:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788204966.748785:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788204966.748787:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748789:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:1.0:1788204966.748790:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000100:00000040:3.0:1788204966.748791:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748796:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204966.748797:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788204966.748800:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748803:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788204966.748805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:3.0:1788204966.748805:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204966.748809:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204966.748811:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788204966.748812:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1788204966.748813:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204966.748816:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788204966.748819:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748821:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204966.748823:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:3.0:1788204966.748824:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788204966.748827:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204966.748830:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204966.748831:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788204966.748831:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1788204966.748834:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204966.748836:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204966.748838:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788204966.748842:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000010:3.0:1788204966.748843:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000800:00000010:1.0:1788204966.748847:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000100:00000001:3.0:1788204966.748847:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204966.748854:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204966.748856:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1788204966.748862:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204966.748866:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204966.748867:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1788204966.748871:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204966.748875:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000100:00000001:3.0:1788204966.748877:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204966.748879:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00080000:3.0:1788204966.748880:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788204966.748884:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204966.748885:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788204966.748886:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1788204966.748886:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204966.748890:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788204966.748893:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26653 00000100:00000200:1.0:1788204966.748894:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204966.748897:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204966.748901:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788204966.748904:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788204966.748909:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204966.748915:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.748921:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204966.748921:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204966.748924:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748928:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788204966.748932:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788204966.748934:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788204966.748937:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788204966.748939:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788204966.748943:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204966.748943:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788204966.748946:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788204966.748948:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1788204966.748950:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748952:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204966.748953:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748955:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788204966.748956:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:1.0:1788204966.748958:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788204966.748959:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788204966.748963:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204966.748965:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204966.748967:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788204966.748968:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788204966.748969:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748973:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788204966.748974:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788204966.748975:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321326853 00000008:00000001:3.0:1788204966.748977:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204966.748981:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204966.748985:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16474e0. 00000100:00000010:3.0:1788204966.748987:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788204966.748992:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788204966.748994:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788204966.748995:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.748996:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788204966.748997:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788204966.749000:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788204966.749003:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749005:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321326853 02000000:00000010:3.0:1788204966.749008:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000010:1.0:1788204966.749011:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647d80. 00000100:00000001:3.0:1788204966.749013:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749018:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00100000:3.0:1788204966.749021:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204966.749023:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204966.749030:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749035:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788204966.749040:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204966.749043:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204966.749045:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204966/real 1788204966] req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204966.749047:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204966.749069:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204966.749075:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26663) 00000400:00000001:1.0:1788204966.749076:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204966.749080:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204966.749082:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204966.749086:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204966.749088:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204966.749116:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204966.749121:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204966.749130:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749137:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204966.749138:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749141:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.749143:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204966.749146:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749153:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204966.749157:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749164:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204966.749167:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749168:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.749170:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749172:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204966.749174:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204966.749176:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204966.749180:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204966.749182:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204966.749186:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204966.749194:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543494144:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788204966.749199:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749201:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.749204:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543494144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749211:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204966.749213:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204966.749215:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204966.749218:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204966.749220:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204966.749222:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204966.749224:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204966.749227:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788204966.749230:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204966.749232:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204966.749233:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788204966.749237:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204966.749238:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749240:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749247:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204966.749249:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749251:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204966.749253:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204966.749254:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204966.749257:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749264:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204966.749265:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.749267:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204966.749269:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204966.749271:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204966.749274:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494272:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204966.749277:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204966.749279:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204966.749280:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.749282:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204966.749285:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204966.749287:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204966.749291:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204966.749293:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204966.749296:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204966.749299:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543494272, portal 10 00000100:00000001:1.0:1788204966.749301:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204966.749302:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788204966.749308:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749314:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204966.749324:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204966.749326:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204966.749329:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543494272, offset 0 00000400:00000200:1.0:1788204966.749333:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204966.749336:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204966.749340:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204966.749343:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204966.749346:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204966.749348:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204966.749350:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204966.749353:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471210:471210:256:4294967295] 192.168.201.118@tcp LPNI seq info [471069:471069:8:4294967295] 00000400:00000200:1.0:1788204966.749370:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204966.749377:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204966.749381:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204966.749384:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204966.749388:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204966.749391:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204966.749395:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204966.749397:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1788204966.749401:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204966.749403:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204966.749406:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204966.749409:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204966.749412:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204966.749414:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204966.749416:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204966.749420:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749428:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.749430:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749437:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749438:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204966.749441:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204966.749445:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788204966.749448:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204966.749450:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321325587 00000100:00000001:1.0:1788204966.749452:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321326853 00000100:00000001:1.0:1788204966.749454:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321325587 00000100:00000001:1.0:1788204966.749457:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321326853 00000100:00000001:1.0:1788204966.749459:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204966.749461:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749463:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204966.749466:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204966/real 1788204966] req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204966.749474:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204966.749476:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204966.749477:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204966.749479:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204966.749481:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204966.749483:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204966.749486:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204966.749495:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749501:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204966.749502:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749504:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.749505:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204966.749509:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749516:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204966.749520:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749526:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204966.749529:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749530:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.749531:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749533:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204966.749535:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204966.749538:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204966.749541:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204966.749543:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204966.749547:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204966.749555:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494272:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204966.749558:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204966.749562:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749574:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204966.749575:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.749577:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204966.749578:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204966.749581:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204966.749583:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494400:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204966.749586:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204966.749588:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204966.749590:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.749591:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204966.749594:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204966.749596:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204966.749599:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204966.749601:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204966.749604:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204966.749607:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543494400, portal 10 00000100:00000001:1.0:1788204966.749609:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204966.749610:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788204966.749615:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749622:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204966.749624:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204966.749626:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204966.749628:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543494400, offset 0 00000400:00000200:1.0:1788204966.749632:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204966.749635:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204966.749639:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204966.749641:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204966.749644:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204966.749646:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204966.749648:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204966.749652:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471211:471211:256:4294967295] 192.168.201.118@tcp LPNI seq info [471070:471070:8:4294967295] 00000400:00000200:1.0:1788204966.749656:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204966.749662:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204966.749666:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204966.749669:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204966.749674:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204966.749676:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204966.749680:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204966.749682:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1788204966.749685:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204966.749687:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204966.749689:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204966.749692:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204966.749695:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204966.749697:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204966.749699:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204966.749702:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749710:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.749713:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749719:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749721:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204966.749724:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204966.749727:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204966.749730:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204966.749732:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321325588 00000100:00000001:1.0:1788204966.749734:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321326853 00000100:00000001:1.0:1788204966.749736:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321325588 00000100:00000001:1.0:1788204966.749738:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321326853 00000100:00000001:1.0:1788204966.749740:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204966.749743:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749745:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204966.749748:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204966/real 1788204966] req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204966.749756:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204966.749757:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204966.749758:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204966.749761:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204966.749762:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204966.749764:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204966.749767:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204966.749775:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749781:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204966.749782:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749783:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.749785:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204966.749788:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749794:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204966.749797:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749804:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204966.749806:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749807:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.749809:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.749810:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204966.749812:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204966.749815:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204966.749818:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204966.749819:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204966.749823:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204966.749830:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494400:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204966.749833:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204966.749836:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749842:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204966.749844:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.749846:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204966.749847:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204966.749849:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204966.749851:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494528:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204966.749854:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204966.749856:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204966.749858:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.749859:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204966.749862:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204966.749864:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204966.749867:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204966.749869:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204966.749872:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204966.749874:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543494528, portal 4 00000100:00000001:1.0:1788204966.749876:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204966.749877:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788204966.749882:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749888:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204966.749890:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204966.749892:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204966.749895:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543494528, offset 0 00000400:00000200:1.0:1788204966.749898:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204966.749901:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204966.749905:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204966.749907:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204966.749909:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204966.749911:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204966.749913:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204966.749916:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471212:471212:256:4294967295] 192.168.201.118@tcp LPNI seq info [471071:471071:8:4294967295] 00000400:00000200:1.0:1788204966.749920:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204966.749926:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204966.749930:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204966.749933:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204966.749936:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204966.749938:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204966.749942:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204966.749945:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1788204966.749948:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204966.749950:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204966.749952:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204966.749956:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204966.749959:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204966.749960:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204966.749963:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204966.749966:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749973:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.749976:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.749982:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.749983:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204966.749986:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204966.749989:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204966.749992:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204966.749994:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321325597 00000100:00000001:1.0:1788204966.749996:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321326853 00000100:00000001:1.0:1788204966.749998:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321325597 00000100:00000001:1.0:1788204966.749999:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321326853 00000100:00000001:1.0:1788204966.750001:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204966.750003:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.750004:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204966.750007:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204966/real 1788204966] req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204966.750014:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204966.750016:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204966.750017:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204966.750020:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204966.750022:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204966.750023:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204966.750026:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204966.750033:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750039:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204966.750040:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750042:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.750043:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204966.750046:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750053:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204966.750057:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750064:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204966.750066:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750068:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.750069:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750070:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204966.750072:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204966.750074:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204966.750078:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204966.750080:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204966.750084:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204966.750110:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494528:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204966.750115:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204966.750118:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750124:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204966.750125:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.750127:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204966.750129:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204966.750131:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204966.750134:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494656:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204966.750137:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204966.750139:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204966.750140:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204966.750142:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204966.750145:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204966.750147:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204966.750150:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204966.750153:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204966.750156:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204966.750159:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543494656, portal 4 00000100:00000001:1.0:1788204966.750161:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204966.750162:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788204966.750167:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750172:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204966.750174:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204966.750176:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 00000100:00000200:1.0:1788204966.750178:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543494656, offset 0 00000400:00000200:1.0:1788204966.750181:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204966.750185:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204966.750188:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204966.750191:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204966.750193:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204966.750195:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204966.750197:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204966.750200:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471213:471213:256:4294967295] 192.168.201.118@tcp LPNI seq info [471072:471072:8:4294967295] 00000400:00000200:1.0:1788204966.750204:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204966.750209:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204966.750213:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204966.750217:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204966.750220:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204966.750222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1788204966.750226:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204966.750229:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1788204966.750232:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204966.750234:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:1.0:1788204966.750237:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204966.750240:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204966.750243:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:1.0:1788204966.750245:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:1.0:1788204966.750247:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204966.750251:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750257:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.750260:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204991 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750266:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.750267:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204966.750270:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204966.750273:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204966.750276:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204966.750277:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321325621 00000100:00000001:1.0:1788204966.750280:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321326853 00000100:00000001:1.0:1788204966.750282:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321325621 00000100:00000001:1.0:1788204966.750284:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321326853 00000100:00000001:1.0:1788204966.750286:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204966.750288:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.750289:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204966.750293:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204966/real 1788204966] req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204966.750300:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204966.750301:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204966.750303:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204966.750305:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204966.750307:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204966.750309:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204966.750312:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204966.750319:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750329:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204966.750331:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750332:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.750334:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204966.750337:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750343:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204966.750346:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750353:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204966.750368:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750370:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204966.750371:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750373:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204966.750374:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204966.750377:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204966.750380:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204966.750382:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204966.750385:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204966.750392:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494656:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204966.750396:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750397:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.750400:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543494272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750405:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204966.750406:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204966.750409:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204966.750411:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204966.750412:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204966.750414:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204966.750415:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204966.750417:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788204966.750419:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204966.750420:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204966.750421:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788204966.750424:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204966.750425:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750426:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.750429:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543494400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750435:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204966.750436:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204966.750437:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204966.750439:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204966.750440:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204966.750441:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204966.750442:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204966.750444:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788204966.750445:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204966.750446:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204966.750448:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788204966.750450:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204966.750451:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750453:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.750456:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543494528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750462:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204966.750463:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204966.750464:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204966.750466:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204966.750467:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204966.750468:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204966.750470:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204966.750472:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788204966.750473:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204966.750474:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204966.750476:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788204966.750478:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204966.750479:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750480:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204966.750483:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543494656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204991 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204966.750489:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204966.750490:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204966.750492:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204966.750494:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204966.750495:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204966.750496:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204966.750498:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204966.750499:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788204966.750501:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204966.750502:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204966.750504:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788204966.750505:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204966.750506:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750508:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204966.750512:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204966.750514:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204966.750515:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204966.750517:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204967.580165:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204967.580165:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204967.580173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204967.580173:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204967.580174:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204967.580175:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204967.580177:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204967.580180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204967.580183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204967.580183:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204967.580188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204967.580189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204967.580189:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204967.580192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204967.580195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204967.580195:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204967.580198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204967.580201:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204967.580202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204967.580202:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204967.580205:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204967.580207:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204967.580209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204967.580210:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204967.580213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204967.580213:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204967.580264:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204967.580268:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204967.580270:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204968.092183:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321326853, at 4321326912 00000100:00000040:3.0:1788204968.092183:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321326853, at 4321326912 00000100:00000010:2.0:1788204968.092196:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16474e0. 00000100:00000010:3.0:1788204968.092198:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00000100:00000001:1.0:1788204968.604178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204968.604178:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204968.604181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204968.604186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204968.604186:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204968.604187:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204968.604189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204968.604189:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204968.604190:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204968.604193:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204968.604196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204968.604196:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204968.604197:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204968.604200:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204968.604202:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204968.604203:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204968.604203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204968.604207:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204968.604207:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204968.604208:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204968.604212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204968.604214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204968.604214:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204968.604215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204968.604218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204968.604218:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204968.604219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204968.604222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204968.604223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204968.604223:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204968.604226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204968.604228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204968.604278:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204968.604281:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204968.604283:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204969.628193:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204969.628193:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204969.628198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204969.628201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204969.628201:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204969.628203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204969.628205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204969.628205:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204969.628212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204969.628212:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204969.628213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204969.628216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204969.628219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204969.628219:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204969.628222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204969.628223:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204969.628224:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204969.628225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204969.628229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204969.628229:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204969.628232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204969.628233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204969.628233:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204969.628238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204969.628239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204969.628239:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204969.628301:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204969.628329:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204969.628331:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204970.652175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204970.652175:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204970.652181:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204970.652182:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204970.652185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.652186:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204970.652192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204970.652192:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204970.652194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204970.652196:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204970.652197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204970.652199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204970.652202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204970.652202:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204970.652204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204970.652207:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204970.652208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204970.652209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204970.652211:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204970.652212:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204970.652215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204970.652215:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204970.652216:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204970.652219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204970.652224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204970.652226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204970.652261:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204970.652265:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204970.652268:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204971.676166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204971.676166:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204971.676175:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204971.676175:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204971.676177:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.676178:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204971.676184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204971.676184:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204971.676190:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204971.676191:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204971.676195:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204971.676195:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204971.676200:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204971.676201:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204971.676204:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204971.676205:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204971.676209:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204971.676209:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204971.676218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204971.676224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204971.676232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204971.676233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204971.676237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204971.676238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204971.676239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204971.676240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204971.676321:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204971.676326:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204971.676331:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204971.868247:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204971.868259:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204971.868261:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204971.868277:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204971.868282:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204971.868289:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26658 00000400:00000200:2.0:1788204971.868301:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204971.868308:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204971.868311:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204971.868314:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868316:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204971.868318:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868320:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204971.868323:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204971.868325:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204971.868327:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204971.868330:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204971.868332:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204971.868334:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204971.868336:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868337:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204971.868339:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868340:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204971.868342:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204971.868344:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868346:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204971.868348:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204971.868352:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204971.868355:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204971.868358:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204971.868361:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204971.868373:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204971.868378:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204971.868382:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204971.868384:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204971.868388:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204971.868392:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204971.868397:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204971.868406:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788204971.868414:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204971.868425:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204971.868436:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204971.868441:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204971.868446:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204971.868470:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204971.868474:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204971.868478:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204971.868480:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204971.868481:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204971.868484:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204971.868487:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26658 00000400:00000200:2.0:1788204971.868493:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204971.868496:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204971.868499:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204971.868503:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.868504:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788204971.868506:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204971.868510:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788204971.868510:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.868514:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204971.868515:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204971.868521:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000040:1.0:1788204971.868522:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204971.868524:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204971.868528:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204971.868532:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:1.0:1788204971.868533:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204971.868536:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204971.868537:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 02000000:00000001:1.0:1788204971.868540:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1788204971.868542:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 02000000:00000001:1.0:1788204971.868543:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204971.868547:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204971.868550:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00100000:1.0:1788204971.868554:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543494784:192.168.201.118@tcp:250:kworker.0 00000100:00000001:2.0:1788204971.868555:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204971.868561:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.868562:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788204971.868566:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:1.0:1788204971.868569:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1788204971.868571:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:1.0:1788204971.868574:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204971.868577:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:1.0:1788204971.868579:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788204971.868583:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204971.868587:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1788204971.868588:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204971.868592:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204971.868594:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204971.868599:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.868600:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204971.868603:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:1.0:1788204971.868606:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000008:00100000:2.0:1788204971.868610:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1788204971.868615:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000008:00000001:2.0:1788204971.868616:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204971.868620:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204971.868624:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000100:00000200:1.0:1788204971.868625:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543494784, portal 25 00000100:00000001:1.0:1788204971.868629:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1788204971.868629:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000001:1.0:1788204971.868631:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331919360 : -109424377632256 : ffff9c7aa6d76a00) 02000000:00000001:2.0:1788204971.868636:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1788204971.868639:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204971.868640:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204971.868645:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204971.868648:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788204971.868652:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788204971.868652:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204971.868656:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204971.868660:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a63b8. 02000000:00000010:2.0:1788204971.868660:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000200:1.0:1788204971.868665:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543494784, offset 0 00000100:00000001:2.0:1788204971.868667:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.868672:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00100000:2.0:1788204971.868678:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204971.868683:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204971.868691:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204971.868696:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204971.868700:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204971.868703:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204971.868705:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868706:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788204971.868710:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471214:471214:256:4294967295] 192.168.201.118@tcp LPNI seq info [471073:471073:8:4294967295] 00000400:00000200:1.0:1788204971.868713:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204971.868719:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.868724:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204971.868732:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000040:2.0:1788204971.868733:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1788204971.868737:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97e00. 00000800:00000200:1.0:1788204971.868748:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204971.868748:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204971.868755:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1788204971.868756:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204971.868763:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204971.868765:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204971.868767:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204971.868770:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204971.868773:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26658 00000400:00000200:2.0:1788204971.868778:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204971.868780:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204971.868785:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204971.868788:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1788204971.868791:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788204971.868795:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.868796:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321331973 00000100:00000001:2.0:1788204971.868802:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:1.0:1788204971.868803:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647900. 00000100:00000040:2.0:1788204971.868806:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868810:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204971.868811:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000020:00000001:2.0:1788204971.868813:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204971.868816:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000100:00000001:1.0:1788204971.868819:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321331973 00000020:00000001:2.0:1788204971.868819:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204971.868822:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000010:1.0:1788204971.868824:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647a80. 00000020:00000001:2.0:1788204971.868826:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788204971.868828:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000020:00000001:2.0:1788204971.868832:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788204971.868833:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204971.868836:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204971.868838:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.868840:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204971.868842:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204971.868844:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204971.868847:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868850:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204971.868852:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000001:1.0:1788204971.868853:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204971.868855:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00080000:2.0:1788204971.868857:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204971.868858:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204971.868861:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204971.868864:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204971.868865:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204971.868868:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204971.868870:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204971.868875:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204971.868876:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204971.868879:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204971.868881:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204971.868881:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204971.868883:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204971.868886:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204971.868886:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1788204971.868891:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494912:192.168.201.118@tcp:38:kworker.0 00000008:00100000:2.0:1788204971.868891:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788204971.868896:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788204971.868898:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204971.868899:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204971.868901:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204971.868903:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788204971.868903:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204971.868907:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204971.868910:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204971.868910:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000400:00000200:1.0:1788204971.868915:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:2.0:1788204971.868916:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000010:1.0:1788204971.868918:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18790. 02000000:00000001:2.0:1788204971.868922:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1788204971.868924:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 02000000:00000001:2.0:1788204971.868925:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788204971.868928:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543494912, portal 10 00000100:00000001:2.0:1788204971.868929:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788204971.868931:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204971.868933:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 02000000:00000001:2.0:1788204971.868933:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204971.868937:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204971.868941:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788204971.868945:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:1.0:1788204971.868946:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204971.868950:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868952:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204971.868953:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6b28. 00000100:00000200:1.0:1788204971.868957:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543494912, offset 0 00000400:00000200:1.0:1788204971.868962:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00100000:2.0:1788204971.868964:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204971.868967:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204971.868972:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204971.868979:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204971.868982:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:2.0:1788204971.868985:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788204971.868986:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204971.868990:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.868990:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.868996:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471215:471215:255:4294967295] 192.168.201.118@tcp LPNI seq info [471074:471074:7:4294967295] 00000100:00000040:2.0:1788204971.869000:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204971.869001:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204971.869014:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.869016:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:2.0:1788204971.869019:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1788204971.869022:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204971.869027:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00080000:2.0:1788204971.869028:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:1.0:1788204971.869032:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204971.869032:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204971.869036:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1788204971.869040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788204971.869042:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788204971.869043:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204971.869047:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000100:00080000:2.0:1788204971.869049:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26658 00000400:00000200:1.0:1788204971.869051:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204971.869055:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321330706 00000100:00000001:1.0:1788204971.869058:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321331973 00000400:00000200:2.0:1788204971.869059:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204971.869062:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321330706 00000100:00000001:1.0:1788204971.869064:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321331973 00000400:00000200:2.0:1788204971.869066:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788204971.869067:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204971.869071:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204971.869072:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788204971.869073:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869077:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204971.869079:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.869081:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788204971.869083:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204971.869085:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788204971.869124:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788204971.869125:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.869126:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204971.869129:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204971.869131:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788204971.869135:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204971.869136:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204971.869139:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204971.869142:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204971.869147:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00000001:1.0:1788204971.869150:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788204971.869151:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788204971.869153:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204971.869156:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 02000000:00000001:1.0:1788204971.869158:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204971.869163:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788204971.869163:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788204971.869169:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204971.869169:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204971.869175:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00100000:1.0:1788204971.869176:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495040:192.168.201.118@tcp:38:kworker.0 00000100:00000001:2.0:1788204971.869180:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204971.869183:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1788204971.869186:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 02000000:00000001:1.0:1788204971.869187:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204971.869191:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204971.869192:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:1.0:1788204971.869194:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788204971.869198:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204971.869199:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:2.0:1788204971.869202:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204971.869204:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204971.869207:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788204971.869209:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788204971.869213:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204971.869215:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18210. 00000100:00000001:2.0:1788204971.869218:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000010:1.0:1788204971.869221:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6cc0. 00000008:00100000:2.0:1788204971.869222:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000200:1.0:1788204971.869227:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543495040, portal 10 00000008:00000001:2.0:1788204971.869228:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869233:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788204971.869234:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869236:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000010:2.0:1788204971.869240:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000100:00000040:1.0:1788204971.869245:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204971.869246:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204971.869252:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204971.869255:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204971.869260:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788204971.869261:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788204971.869275:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204971.869279:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204971.869283:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000010:2.0:1788204971.869285:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000400:00000010:1.0:1788204971.869287:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6bb0. 00000100:00000200:1.0:1788204971.869292:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543495040, offset 0 00000100:00000001:2.0:1788204971.869292:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.869303:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00100000:2.0:1788204971.869303:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788204971.869311:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204971.869318:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:2.0:1788204971.869322:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788204971.869323:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788204971.869327:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.869328:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204971.869333:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1788204971.869337:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204971.869339:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204971.869346:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471216:471216:254:4294967295] 192.168.201.118@tcp LPNI seq info [471075:471075:6:4294967295] 00000400:00000200:1.0:1788204971.869357:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204971.869359:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204971.869365:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788204971.869368:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:2.0:1788204971.869372:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788204971.869376:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.0:1788204971.869377:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788204971.869379:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:1.0:1788204971.869384:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97c00. 00000100:00000001:2.0:1788204971.869385:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.0:1788204971.869392:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204971.869393:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26658 00000800:00000200:1.1:1788204971.869402:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:2.0:1788204971.869403:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204971.869409:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788204971.869411:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000040:1.0:1788204971.869414:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000100:00080000:2.0:1788204971.869416:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1788204971.869422:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788204971.869423:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.869427:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321330708 00000100:00000001:2.0:1788204971.869428:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204971.869433:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321331973 00000100:00000040:2.0:1788204971.869433:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204971.869436:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204971.869439:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321330708 00000020:00000001:2.0:1788204971.869442:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:1.0:1788204971.869445:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321331973 00000020:00000040:2.0:1788204971.869446:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204971.869449:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788204971.869451:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1788204971.869453:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788204971.869455:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204971.869458:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788204971.869460:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204971.869463:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788204971.869465:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204971.869472:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.869473:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204971.869476:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788204971.869477:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788204971.869479:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788204971.869482:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204971.869483:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788204971.869486:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788204971.869487:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1788204971.869491:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000040:1.0:1788204971.869494:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204971.869498:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204971.869505:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204971.869509:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788204971.869511:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204971.869514:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204971.869515:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204971.869517:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204971.869520:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204971.869521:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204971.869524:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788204971.869525:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788204971.869529:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788204971.869531:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495168:192.168.201.118@tcp:8:kworker.0 00000008:00000001:2.0:1788204971.869534:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869539:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788204971.869540:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204971.869542:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204971.869546:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204971.869547:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 02000000:00000001:1.0:1788204971.869549:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1788204971.869552:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000010:1.0:1788204971.869555:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:2.0:1788204971.869558:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204971.869559:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204971.869561:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.869566:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204971.869566:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204971.869569:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1788204971.869572:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18478. 02000000:00000001:2.0:1788204971.869573:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788204971.869578:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a65d8. 02000000:00000010:2.0:1788204971.869583:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000200:1.0:1788204971.869585:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543495168, portal 4 00000100:00000001:1.0:1788204971.869589:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204971.869590:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000100:00000001:2.0:1788204971.869592:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204971.869600:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788204971.869602:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204971.869620:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204971.869625:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204971.869625:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1788204971.869630:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6dd0. 00000100:00000200:1.0:1788204971.869633:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543495168, offset 0 00000100:00000001:2.0:1788204971.869635:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.869639:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204971.869645:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788204971.869645:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788204971.869652:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204971.869660:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204971.869664:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:2.0:1788204971.869666:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.869669:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204971.869674:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000040:2.0:1788204971.869675:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26668) 00000400:00000200:1.0:1788204971.869678:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471217:471217:253:4294967295] 192.168.201.118@tcp LPNI seq info [471076:471076:5:4294967295] 00000400:00000200:1.0:1788204971.869683:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204971.869689:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204971.869694:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204971.869698:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97300. 00000800:00000200:1.0:1788204971.869703:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204971.869707:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788204971.869709:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204971.869712:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204971.869716:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204971.869718:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321330717 00000100:00000001:1.0:1788204971.869721:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321331973 00000100:00000001:1.0:1788204971.869724:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321330717 00000100:00000001:1.0:1788204971.869726:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321331973 00000100:00000001:1.0:1788204971.869728:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204971.869730:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869733:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869735:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204971.869738:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204971.869740:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788204971.869742:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204971.869744:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204971.869745:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204971.869750:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204971.869758:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204971.869759:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204971.869761:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204971.869763:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204971.869765:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204971.869768:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495296:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204971.869771:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204971.869773:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204971.869774:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204971.869776:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204971.869780:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204971.869782:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204971.869785:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204971.869788:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788204971.869792:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a64c8. 00000100:00000200:1.0:1788204971.869795:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543495296, portal 4 00000100:00000001:1.0:1788204971.869798:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204971.869799:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:1.0:1788204971.869806:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204971.869813:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204971.869816:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204971.869819:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a62a8. 00000100:00000200:1.0:1788204971.869822:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543495296, offset 0 00000400:00000200:1.0:1788204971.869827:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204971.869830:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204971.869833:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204971.869836:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204971.869838:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204971.869841:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204971.869843:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204971.869846:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471218:471218:252:4294967295] 192.168.201.118@tcp LPNI seq info [471077:471077:4:4294967295] 00000400:00000200:1.0:1788204971.869850:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204971.869855:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204971.869859:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204971.869863:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97200. 00000800:00000200:1.0:1788204971.869866:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204971.869870:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788204971.869874:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204971.869878:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204971.869880:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204971.869883:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321330740 00000100:00000001:1.0:1788204971.869886:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321331973 00000100:00000001:1.0:1788204971.869888:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321330740 00000100:00000001:1.0:1788204971.869890:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321331973 00000100:00000001:1.0:1788204971.869893:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204971.869894:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869897:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869899:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204971.869902:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204971.869903:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788204971.869905:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204971.869906:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204971.869908:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204971.869910:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204972.700158:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204972.700158:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204972.700166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204972.700166:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204972.700167:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204972.700171:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204972.700171:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204972.700171:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204972.700179:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204972.700179:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204972.700180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204972.700183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204972.700186:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204972.700186:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204972.700187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204972.700190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204972.700191:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204972.700191:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204972.700193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204972.700196:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204972.700197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204972.700197:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204972.700201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204972.700201:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204972.700207:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204972.700207:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204972.700285:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204972.700289:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204972.700292:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204973.212185:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321331973, at 4321332032 00000100:00000040:3.0:1788204973.212185:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321331973, at 4321332032 00000100:00000010:3.0:1788204973.212199:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00000100:00000010:2.0:1788204973.212201:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647900. 00000100:00000001:1.0:1788204973.724197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204973.724197:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204973.724200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204973.724205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204973.724206:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204973.724206:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204973.724210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204973.724210:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204973.724216:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:1.0:1788204973.724216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204973.724216:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204973.724218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204973.724221:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204973.724223:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204973.724223:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204973.724224:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204973.724226:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204973.724227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204973.724227:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204973.724232:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204973.724232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204973.724232:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204973.724248:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204973.724249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204973.724253:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204973.724259:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204973.724260:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204973.724262:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204973.724262:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204973.724267:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788204973.724268:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724269:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204973.724281:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204973.724282:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204973.724283:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724284:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204973.724285:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724286:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204973.724287:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204973.724346:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204973.724351:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204973.724354:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204974.748236:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204974.748238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204974.748243:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204974.748245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204974.748246:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204974.748252:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204974.748253:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204974.748255:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204974.748257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204974.748264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204974.748265:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204974.748266:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204974.748275:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204974.748277:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204974.748279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204974.748280:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.748285:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204974.748287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204974.748287:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204974.748293:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.748295:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204974.748296:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204974.748298:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204974.748300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204974.748301:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.748311:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204974.748347:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204974.748349:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204974.748351:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788204974.940442:0:256122:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:3.0:1788204974.940513:0:256122:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1788204974.940525:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204974.940529:0:256122:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:3.0:1788204974.940534:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204974.940539:0:256122:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:3.0:1788204974.940545:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:3.0:1788204974.940550:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97e00. 00000400:00000200:3.0:1788204974.940557:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788204974.940563:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788204974.940572:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a63b8 00000400:00000010:3.0:1788204974.940575:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a63b8. 00000100:00000001:3.0:1788204974.940582:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204974.940594:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940629:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204974.940634:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940641:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204974.940643:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1788204974.940656:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:3.0:1788204974.940659:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:3.0:1788204974.940662:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788204974.940666:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788204974.940669:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6b28 00000400:00000010:3.0:1788204974.940671:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6b28. 00000100:00000001:3.0:1788204974.940673:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204974.940678:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940687:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204974.940690:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940697:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204974.940699:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1788204974.940705:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:3.0:1788204974.940715:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000400:00000200:3.0:1788204974.940719:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788204974.940722:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788204974.940725:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6bb0 00000400:00000010:3.0:1788204974.940727:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6bb0. 00000100:00000001:3.0:1788204974.940730:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204974.940736:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940744:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204974.940746:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940752:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204974.940754:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1788204974.940759:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:3.0:1788204974.940762:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000400:00000200:3.0:1788204974.940765:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788204974.940768:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788204974.940771:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6dd0 00000400:00000010:3.0:1788204974.940773:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6dd0. 00000100:00000001:3.0:1788204974.940776:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204974.940781:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940789:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204974.940792:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940798:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204974.940800:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1788204974.940805:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:3.0:1788204974.940808:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:3.0:1788204974.940812:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788204974.940816:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1788204974.940819:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a62a8 00000400:00000010:3.0:1788204974.940821:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a62a8. 00000100:00000001:3.0:1788204974.940824:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204974.940829:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940836:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204974.940839:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788204996 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204974.940844:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204974.940846:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204974.940954:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204974.940961:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204974.940965:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204974.940976:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204971/real 1788204974] req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204974.940993:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204974.940995:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204974.940999:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788204974.941003:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a64c8 00000400:00000010:1.0:1788204974.941005:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a64c8. 00000100:00000001:1.0:1788204974.941009:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204974.941013:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204974.941023:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941030:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204974.941032:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941035:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941038:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204974.941041:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941048:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204974.941054:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941061:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204974.941064:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941066:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941067:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941070:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204974.941073:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204974.941077:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204974.941083:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204974.941085:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204974.941115:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204974.941124:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495296:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204974.941129:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204974.941132:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204971/real 1788204974] req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204974.941140:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204974.941141:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204974.941143:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18478. 00000400:00000200:1.0:1788204974.941146:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a65d8 00000400:00000010:1.0:1788204974.941148:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a65d8. 00000100:00000001:1.0:1788204974.941150:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204974.941154:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204974.941161:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941167:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204974.941169:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941171:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941172:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204974.941175:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941182:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204974.941185:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941191:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204974.941193:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941195:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941196:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941197:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204974.941200:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204974.941202:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204974.941205:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204974.941207:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204974.941211:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204974.941229:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495168:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204974.941234:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204974.941238:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204971/real 1788204974] req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204974.941253:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204974.941255:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204974.941259:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18210. 00000400:00000200:1.0:1788204974.941263:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6cc0 00000400:00000010:1.0:1788204974.941266:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6cc0. 00000100:00000001:1.0:1788204974.941269:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204974.941272:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204974.941280:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941286:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204974.941287:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941289:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941290:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204974.941295:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941302:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204974.941306:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941326:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204974.941329:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941331:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941333:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941334:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204974.941338:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204974.941340:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204974.941345:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204974.941348:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204974.941352:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204974.941360:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495040:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204974.941365:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204974.941367:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204971/real 1788204974] req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204974.941375:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204974.941376:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204974.941378:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18790. 00000400:00000200:1.0:1788204974.941381:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788204974.941383:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788204974.941385:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204974.941388:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204974.941396:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941404:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204974.941405:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941407:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941409:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204974.941413:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941421:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204974.941425:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941432:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204974.941434:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941437:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941439:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941440:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204974.941444:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204974.941447:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204974.941453:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204974.941455:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204974.941459:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204974.941467:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543494912:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204974.941472:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204974.941476:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204971/real 1788204974] req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204974.941486:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204974.941487:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204974.941489:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204974.941494:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204974.941496:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204974.941498:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204974.941503:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204974.941511:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941518:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204974.941520:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941523:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941524:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204974.941528:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941541:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204974.941545:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941552:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204974.941555:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941557:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204974.941558:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941560:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204974.941563:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204974.941566:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204974.941571:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204974.941574:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204974.941580:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204974.941588:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543494784:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788204974.941593:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941596:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204974.941599:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543495296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941607:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204974.941611:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204974.941616:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204974.941622:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204974.941626:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204974.941628:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204974.941631:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204974.941634:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:1.0:1788204974.941639:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204974.941642:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204974.941644:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788204974.941649:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204974.941651:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941653:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204974.941657:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543495168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941663:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204974.941665:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204974.941666:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204974.941669:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204974.941670:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204974.941671:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204974.941674:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204974.941676:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e800. 02000000:00000001:1.0:1788204974.941679:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204974.941681:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204974.941682:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:1.0:1788204974.941686:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204974.941687:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941689:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204974.941702:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543495040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941708:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204974.941709:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204974.941711:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204974.941714:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204974.941715:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204974.941716:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204974.941719:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204974.941720:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d000. 02000000:00000001:1.0:1788204974.941723:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204974.941724:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204974.941725:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:1.0:1788204974.941728:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204974.941730:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941732:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204974.941735:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543494912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941742:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204974.941743:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204974.941745:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204974.941747:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204974.941748:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204974.941749:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204974.941751:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204974.941752:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:1.0:1788204974.941753:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204974.941754:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204974.941755:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:1.0:1788204974.941756:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204974.941757:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941758:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204974.941760:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76a00 x1875055543494784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788204996 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204974.941764:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204974.941764:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204974.941765:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204974.941767:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204974.941768:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204974.941769:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204974.941770:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204974.941771:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788204974.941773:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204974.941774:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204974.941775:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76a00. 00000100:00000001:1.0:1788204974.941777:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204974.941778:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941780:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204974.941792:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204974.941794:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204974.941797:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204974.941799:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.772222:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204975.772223:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204975.772230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204975.772235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.772238:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204975.772239:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204975.772242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204975.772243:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204975.772244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204975.772247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204975.772256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204975.772257:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204975.772258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204975.772259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204975.772270:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204975.772270:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204975.772271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204975.772275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204975.772276:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204975.772277:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204975.772282:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204975.772283:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204975.772285:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204975.772289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204975.772301:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204975.772305:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204975.772347:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204975.772351:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204975.772355:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204976.796233:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204976.796233:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204976.796244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204976.796244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204976.796244:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204976.796250:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.796251:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204976.796252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204976.796261:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204976.796261:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204976.796264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204976.796269:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204976.796273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.796274:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204976.796275:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204976.796284:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204976.796285:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204976.796287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204976.796287:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.796296:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204976.796297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204976.796298:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.796308:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.796312:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204976.796320:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204976.796322:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204976.796395:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204976.796399:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204976.796402:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788204976.988226:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.988237:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788204976.988240:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204976.988243:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204976.988247:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204976.988254:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26663 00000400:00000200:0.0:1788204976.988264:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204976.988270:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204976.988273:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204976.988275:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988278:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.988279:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988281:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204976.988285:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204976.988287:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788204976.988289:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204976.988292:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788204976.988294:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788204976.988297:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788204976.988300:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988302:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.988432:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988438:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204976.988443:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788204976.988449:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988452:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204976.988456:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204976.988462:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204976.988464:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988467:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788204976.988470:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204976.988481:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788204976.988486:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788204976.988490:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204976.988492:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988495:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204976.988499:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204976.988504:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204976.988512:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788204976.988520:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204976.988531:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204976.988542:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204976.988548:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204976.988553:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204976.988592:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204976.988597:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.988601:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788204976.988603:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204976.988606:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204976.988610:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204976.988615:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26663 00000400:00000200:0.0:1788204976.988630:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204976.988638:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204976.988642:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204976.988646:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988648:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.988651:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988652:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204976.988656:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204976.988657:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788204976.988660:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204976.988664:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788204976.988666:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788204976.988668:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788204976.988671:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988673:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.988675:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988677:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204976.988679:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788204976.988681:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988684:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204976.988686:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204976.988693:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204976.988695:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988696:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204976.988700:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204976.988703:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988705:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204976.988708:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788204976.988711:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204976.988713:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204976.988715:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988716:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204976.988718:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204976.988719:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204976.988722:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788204976.988726:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204976.988731:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204976.988737:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204976.988739:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204976.988744:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204976.988751:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204976.988753:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.988756:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788204976.988757:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204976.988759:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204976.988761:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204976.988764:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26663 00000400:00000200:0.0:1788204976.988768:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204976.988770:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204976.988772:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204976.988775:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988777:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.988779:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988781:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204976.988783:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204976.988784:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788204976.988786:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204976.988787:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788204976.988789:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788204976.988792:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788204976.988795:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988797:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.988799:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988801:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204976.988803:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788204976.988805:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988807:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204976.988809:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204976.988812:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204976.988814:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988816:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204976.988817:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788204976.988819:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988821:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204976.988824:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788204976.988826:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788204976.988829:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204976.988830:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988831:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204976.988832:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204976.988834:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204976.988838:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788204976.988841:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204976.988845:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204976.988851:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204976.988854:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204976.988857:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204976.988864:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204976.988867:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.988870:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788204976.988871:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204976.988873:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204976.988876:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204976.988878:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26663 00000400:00000200:0.0:1788204976.988882:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204976.988884:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204976.988886:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204976.988889:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988891:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.988892:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988894:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204976.988895:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204976.988896:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788204976.988898:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204976.988900:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788204976.988901:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788204976.988903:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788204976.988905:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988907:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.988908:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988910:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204976.988913:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788204976.988915:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204976.988917:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204976.988918:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204976.988921:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204976.988922:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988924:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204976.988927:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788204976.988929:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988930:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204976.988933:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788204976.988935:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204976.988937:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204976.988938:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.988940:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204976.988941:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204976.988943:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204976.988946:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788204976.988950:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204976.988953:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204976.988959:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204976.988961:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204976.988964:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204976.988972:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788204976.988974:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788204976.988978:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788204976.988980:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788204976.988981:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788204976.988985:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788204976.988987:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26663 00000400:00000200:0.0:1788204976.988991:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788204976.988994:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788204976.988996:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788204976.988998:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.989001:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.989003:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.989005:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788204976.989007:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788204976.989009:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788204976.989011:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788204976.989012:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788204976.989014:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788204976.989016:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788204976.989019:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.989020:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788204976.989022:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788204976.989024:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788204976.989027:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788204976.989030:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788204976.989032:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788204976.989034:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788204976.989037:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788204976.989038:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.989040:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788204976.989043:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788204976.989045:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.989046:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788204976.989049:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788204976.989052:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788204976.989054:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788204976.989055:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788204976.989057:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788204976.989059:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788204976.989060:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788204976.989063:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788204976.989066:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788204976.989071:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788204976.989084:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788204976.989085:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204976.989088:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788204976.989153:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788204976.989158:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26673) 00000100:00000001:1.0:1788204976.989871:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204976.989878:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204976.989884:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204976.989894:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.989904:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204976.989907:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.989911:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204976.989914:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204976.989917:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204976.989921:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543495424:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788204976.989925:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204976.989931:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204976.989935:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.989937:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204976.989946:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204976.989948:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204976.989955:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204976.989959:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204976.989967:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204976.989975:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543495424, portal 25 00000100:00000001:1.0:1788204976.989977:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204976.989978:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904111360 : -109419805440256 : ffff9c7bb75d8700) 00000100:00000040:1.0:1788204976.989984:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.989991:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204976.989995:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204976.989997:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788204976.990000:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543495424, offset 0 00000400:00000200:1.0:1788204976.990005:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990013:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990018:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204976.990023:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204976.990027:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204976.990030:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204976.990032:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990035:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471219:471219:256:4294967295] 192.168.201.118@tcp LPNI seq info [471078:471078:8:4294967295] 00000400:00000200:1.0:1788204976.990039:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990046:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204976.990051:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990055:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788204976.990061:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204976.990064:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204976.990068:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204976.990071:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788204976.990075:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990077:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788204976.990082:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204976.990086:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204976.990124:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788204976.990127:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788204976.990130:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204976.990133:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990140:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.990143:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990148:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990149:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204976.990155:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204976.990157:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204976.990160:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321337095 00000100:00000010:1.0:1788204976.990163:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647660. 00000100:00000001:1.0:1788204976.990170:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788204976.990174:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321337095 00000100:00000010:1.0:1788204976.990176:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647840. 00000100:00000001:1.0:1788204976.990197:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788204976.990199:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204976.990201:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990203:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204976.990206:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204976.990213:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204976.990214:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204976.990216:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204976.990218:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204976.990219:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204976.990221:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204976.990223:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204976.990228:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990232:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204976.990234:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990235:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990237:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204976.990239:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990243:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204976.990249:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990254:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204976.990256:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990257:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990257:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990259:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204976.990260:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204976.990262:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204976.990264:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204976.990266:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204976.990268:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204976.990273:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543495424:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788204976.990276:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204976.990278:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990282:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204976.990283:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.990284:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204976.990286:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204976.990287:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204976.990289:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495552:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204976.990291:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204976.990292:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204976.990293:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.990294:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204976.990296:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204976.990298:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204976.990300:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204976.990302:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204976.990304:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204976.990305:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543495552, portal 10 00000100:00000001:1.0:1788204976.990307:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204976.990308:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904109568 : -109419805442048 : ffff9c7bb75d8000) 00000100:00000040:1.0:1788204976.990312:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990316:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204976.990317:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204976.990319:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788204976.990320:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543495552, offset 0 00000400:00000200:1.0:1788204976.990322:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990325:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990327:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204976.990328:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204976.990330:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204976.990332:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204976.990333:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990335:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471220:471220:256:4294967295] 192.168.201.118@tcp LPNI seq info [471079:471079:8:4294967295] 00000400:00000200:1.0:1788204976.990338:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990342:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204976.990344:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990346:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788204976.990349:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204976.990350:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204976.990353:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204976.990354:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788204976.990357:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990358:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788204976.990360:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204976.990362:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204976.990364:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788204976.990365:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788204976.990366:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204976.990368:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990373:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.990374:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990378:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990379:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204976.990380:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204976.990386:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788204976.990388:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204976.990390:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321335828 00000100:00000001:1.0:1788204976.990391:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321337095 00000100:00000001:1.0:1788204976.990392:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321335828 00000100:00000001:1.0:1788204976.990394:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321337095 00000100:00000001:1.0:1788204976.990395:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204976.990396:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990397:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204976.990399:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204976.990407:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204976.990408:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204976.990409:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204976.990410:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204976.990411:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204976.990413:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204976.990414:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204976.990419:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990423:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204976.990424:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990424:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990425:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204976.990427:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990431:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204976.990432:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990436:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204976.990437:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990438:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990439:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990439:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204976.990441:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204976.990442:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204976.990444:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204976.990445:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204976.990448:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204976.990453:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495552:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204976.990455:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204976.990457:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990461:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204976.990461:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.990463:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204976.990464:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204976.990465:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204976.990468:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495680:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204976.990471:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204976.990472:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204976.990474:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.990475:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204976.990478:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204976.990480:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204976.990483:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204976.990485:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204976.990488:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204976.990491:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543495680, portal 10 00000100:00000001:1.0:1788204976.990492:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204976.990494:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904122112 : -109419805429504 : ffff9c7bb75db100) 00000100:00000040:1.0:1788204976.990499:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990505:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204976.990507:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204976.990509:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788204976.990511:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543495680, offset 0 00000400:00000200:1.0:1788204976.990516:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990527:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990532:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204976.990533:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204976.990535:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204976.990537:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204976.990538:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990540:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471221:471221:256:4294967295] 192.168.201.118@tcp LPNI seq info [471080:471080:8:4294967295] 00000400:00000200:1.0:1788204976.990542:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990546:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204976.990549:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990551:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788204976.990554:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204976.990555:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204976.990558:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204976.990559:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788204976.990562:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990563:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788204976.990565:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204976.990566:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204976.990568:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788204976.990569:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788204976.990571:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204976.990573:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990578:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.990580:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990583:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990584:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204976.990586:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204976.990588:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788204976.990590:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204976.990592:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321335829 00000100:00000001:1.0:1788204976.990593:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321337095 00000100:00000001:1.0:1788204976.990594:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321335829 00000100:00000001:1.0:1788204976.990596:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321337095 00000100:00000001:1.0:1788204976.990597:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204976.990598:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990599:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204976.990601:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204976.990605:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204976.990606:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204976.990607:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204976.990608:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204976.990609:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204976.990610:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204976.990612:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204976.990616:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990619:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204976.990620:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990621:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990622:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204976.990624:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990627:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204976.990629:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990633:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204976.990634:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990635:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990636:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990637:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204976.990638:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204976.990640:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204976.990641:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204976.990643:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204976.990645:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204976.990649:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495680:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788204976.990651:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204976.990653:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990657:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204976.990657:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.990659:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204976.990660:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204976.990661:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204976.990662:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495808:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204976.990664:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204976.990665:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204976.990666:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.990667:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204976.990669:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204976.990670:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204976.990672:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204976.990674:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204976.990677:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204976.990680:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543495808, portal 4 00000100:00000001:1.0:1788204976.990682:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204976.990683:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904110464 : -109419805441152 : ffff9c7bb75d8380) 00000100:00000040:1.0:1788204976.990688:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990693:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204976.990695:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204976.990697:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788204976.990700:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543495808, offset 0 00000400:00000200:1.0:1788204976.990703:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990706:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990709:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204976.990712:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204976.990714:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204976.990716:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204976.990717:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990719:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471222:471222:256:4294967295] 192.168.201.118@tcp LPNI seq info [471081:471081:8:4294967295] 00000400:00000200:1.0:1788204976.990722:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990726:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204976.990729:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990731:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788204976.990733:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204976.990735:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204976.990737:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204976.990739:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788204976.990741:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990742:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788204976.990744:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204976.990746:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204976.990748:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788204976.990749:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788204976.990750:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204976.990752:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990761:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.990762:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990766:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990767:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204976.990769:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204976.990770:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788204976.990772:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204976.990773:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321335838 00000100:00000001:1.0:1788204976.990775:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321337095 00000100:00000001:1.0:1788204976.990776:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321335838 00000100:00000001:1.0:1788204976.990777:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321337095 00000100:00000001:1.0:1788204976.990778:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204976.990780:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990781:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204976.990783:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204976.990786:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204976.990787:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204976.990788:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204976.990789:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204976.990790:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204976.990791:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204976.990793:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204976.990798:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990801:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204976.990802:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990803:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990804:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204976.990806:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990809:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204976.990811:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990814:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204976.990816:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990816:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990817:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990818:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204976.990819:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204976.990821:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204976.990823:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204976.990824:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204976.990826:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204976.990830:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495808:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204976.990833:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204976.990836:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990842:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204976.990844:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.990846:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788204976.990847:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204976.990849:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204976.990853:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495936:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204976.990855:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788204976.990857:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788204976.990859:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204976.990861:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788204976.990863:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204976.990865:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204976.990868:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788204976.990870:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788204976.990873:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788204976.990876:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543495936, portal 4 00000100:00000001:1.0:1788204976.990878:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204976.990879:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904121216 : -109419805430400 : ffff9c7bb75dad80) 00000100:00000040:1.0:1788204976.990883:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990887:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788204976.990888:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788204976.990889:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788204976.990891:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543495936, offset 0 00000400:00000200:1.0:1788204976.990893:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990896:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990898:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788204976.990900:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788204976.990901:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788204976.990903:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788204976.990904:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788204976.990906:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471223:471223:256:4294967295] 192.168.201.118@tcp LPNI seq info [471082:471082:8:4294967295] 00000400:00000200:1.0:1788204976.990908:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204976.990912:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788204976.990914:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990916:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788204976.990918:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788204976.990920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788204976.990922:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788204976.990924:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788204976.990926:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788204976.990927:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788204976.990929:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788204976.990931:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788204976.990933:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788204976.990933:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788204976.990935:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788204976.990937:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990941:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.990943:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205001 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990946:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990947:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788204976.990949:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204976.990951:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788204976.990952:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204976.990954:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321335861 00000100:00000001:1.0:1788204976.990955:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321337095 00000100:00000001:1.0:1788204976.990956:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321335861 00000100:00000001:1.0:1788204976.990958:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321337095 00000100:00000001:1.0:1788204976.990959:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204976.990960:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.990961:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788204976.990963:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204976/real 1788204976] req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788204976.990967:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788204976.990968:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788204976.990969:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788204976.990970:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788204976.990971:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788204976.990972:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788204976.990974:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788204976.990978:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990981:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788204976.990982:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990983:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990983:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204976.990985:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990989:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788204976.990991:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.990994:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788204976.990995:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990996:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788204976.990997:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.990998:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788204976.990999:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788204976.991000:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788204976.991002:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788204976.991003:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788204976.991005:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204976.991013:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543495936:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788204976.991015:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.991017:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.991018:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8700 x1875055543495424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.991022:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204976.991024:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204976.991026:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788204976.991027:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204976.991028:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204976.991029:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788204976.991031:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204976.991033:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:1.0:1788204976.991035:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204976.991036:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204976.991037:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8700. 00000100:00000001:1.0:1788204976.991039:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204976.991040:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.991041:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.991043:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8000 x1875055543495552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.991046:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204976.991047:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204976.991048:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788204976.991049:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204976.991049:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204976.991050:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204976.991051:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204976.991052:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29c00. 02000000:00000001:1.0:1788204976.991053:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204976.991053:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204976.991054:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8000. 00000100:00000001:1.0:1788204976.991056:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204976.991056:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.991058:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.991059:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db100 x1875055543495680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.991063:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204976.991064:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204976.991064:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788204976.991065:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204976.991066:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204976.991067:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788204976.991067:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204976.991068:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28800. 02000000:00000001:1.0:1788204976.991069:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204976.991070:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204976.991071:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db100. 00000100:00000001:1.0:1788204976.991072:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204976.991072:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.991073:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.991075:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8380 x1875055543495808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.991078:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204976.991078:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204976.991079:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788204976.991080:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204976.991081:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204976.991081:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204976.991082:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204976.991083:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29400. 02000000:00000001:1.0:1788204976.991084:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204976.991084:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204976.991085:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8380. 00000100:00000001:1.0:1788204976.991086:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204976.991087:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.991087:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204976.991104:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75dad80 x1875055543495936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205001 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204976.991109:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788204976.991110:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788204976.991111:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788204976.991112:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788204976.991113:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788204976.991113:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788204976.991115:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788204976.991115:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:1.0:1788204976.991117:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788204976.991117:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788204976.991118:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dad80. 00000100:00000001:1.0:1788204976.991120:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788204976.991120:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.991122:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204976.991127:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204976.991128:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204976.991129:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788204976.991130:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204977.820234:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204977.820234:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204977.820236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204977.820243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204977.820245:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204977.820245:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204977.820249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204977.820251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204977.820254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204977.820255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204977.820255:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204977.820260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204977.820260:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204977.820262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204977.820264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204977.820264:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204977.820267:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204977.820268:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204977.820271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204977.820272:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204977.820273:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204977.820274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204977.820276:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204977.820277:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204977.820280:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204977.820282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204977.820355:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204977.820360:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204977.820364:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204978.332240:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321337095, at 4321337152 00000100:00000040:3.0:1788204978.332240:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321337095, at 4321337152 00000100:00000010:2.0:1788204978.332262:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647660. 00000100:00000010:3.0:1788204978.332264:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00000100:00000001:1.0:1788204978.844214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204978.844214:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204978.844225:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204978.844226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204978.844230:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204978.844230:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204978.844235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204978.844239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204978.844239:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204978.844243:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204978.844247:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204978.844247:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204978.844248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788204978.844254:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.844254:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204978.844255:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204978.844259:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204978.844261:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204978.844262:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788204978.844264:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204978.844264:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204978.844267:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204978.844271:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204978.844272:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204978.844272:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204978.844278:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204978.844284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204978.844286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204978.844289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204978.844290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204978.844294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204978.844295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204978.844329:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204978.844334:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204978.844338:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204979.868223:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204979.868223:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204979.868232:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204979.868233:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204979.868234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204979.868234:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204979.868236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204979.868241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204979.868241:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204979.868242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204979.868246:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204979.868248:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204979.868250:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204979.868251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204979.868252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204979.868254:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204979.868255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204979.868257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204979.868260:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204979.868261:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204979.868262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204979.868267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204979.868268:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204979.868268:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204979.868273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204979.868276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204979.868343:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204979.868348:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204979.868352:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204980.892172:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204980.892175:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204980.892181:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204980.892182:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204980.892184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204980.892184:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204980.892185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204980.892189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204980.892192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204980.892193:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204980.892194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204980.892198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204980.892199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204980.892200:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204980.892201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204980.892204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204980.892205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204980.892205:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204980.892207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204980.892211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204980.892212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204980.892212:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204980.892216:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204980.892217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204980.892223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204980.892224:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204980.892277:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204980.892281:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204980.892284:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204981.916216:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204981.916216:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204981.916223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204981.916225:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204981.916226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204981.916229:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204981.916230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204981.916234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204981.916237:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204981.916239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204981.916241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204981.916242:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204981.916244:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204981.916245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204981.916247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204981.916248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204981.916250:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204981.916252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204981.916252:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204981.916254:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204981.916256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204981.916257:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204981.916258:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204981.916264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204981.916264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204981.916267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204981.916334:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204981.916337:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204981.916340:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204982.108233:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204982.108245:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204982.108247:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204982.108251:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204982.108255:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204982.108262:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26668 00000400:00000200:1.0:1788204982.108272:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204982.108278:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204982.108281:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204982.108283:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108286:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204982.108288:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108289:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204982.108293:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204982.108294:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788204982.108297:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204982.108300:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788204982.108301:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788204982.108303:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788204982.108305:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108306:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204982.108307:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108309:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204982.108311:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788204982.108313:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108315:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204982.108317:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204982.108320:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204982.108322:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.108323:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204982.108326:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204982.108336:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788204982.108339:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788204982.108342:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204982.108343:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.108346:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204982.108349:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204982.108354:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204982.108361:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000001:1.0:1788204982.108368:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204982.108376:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204982.108385:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204982.108389:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204982.108392:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204982.108419:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204982.108423:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204982.108427:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204982.108428:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204982.108430:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204982.108433:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204982.108436:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26668 00000400:00000200:1.0:1788204982.108441:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204982.108445:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204982.108447:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204982.108449:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108451:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204982.108452:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1788204982.108453:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108457:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204982.108458:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788204982.108459:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204982.108461:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:3.0:1788204982.108462:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1788204982.108463:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204982.108466:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788204982.108469:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788204982.108471:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204982.108471:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204982.108478:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108482:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204982.108485:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108489:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204982.108489:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1788204982.108493:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:3.0:1788204982.108493:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204982.108498:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788204982.108500:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788204982.108501:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204982.108504:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204982.108504:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204982.108509:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204982.108511:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204982.108515:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204982.108516:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543496064:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788204982.108517:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204982.108522:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788204982.108524:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204982.108530:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:1.0:1788204982.108533:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.108535:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.108536:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204982.108541:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 02000000:00000001:3.0:1788204982.108541:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:1.0:1788204982.108545:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788204982.108547:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204982.108548:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.108550:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:3.0:1788204982.108550:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788204982.108551:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204982.108554:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.108555:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204982.108560:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8800. 00000100:00000001:1.0:1788204982.108564:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.108564:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:1.0:1788204982.108571:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788204982.108576:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:1.0:1788204982.108579:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204982.108583:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204982.108586:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000040:1.0:1788204982.108588:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204982.108594:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543496064, portal 25 00000100:00000001:3.0:1788204982.108599:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788204982.108600:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.108601:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00080000:1.0:1788204982.108608:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1788204982.108611:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204982.108615:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788204982.108619:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204982.108623:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788204982.108623:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788204982.108627:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:3.0:1788204982.108628:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204982.108632:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00080000:1.0:1788204982.108633:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26668 00000100:00000200:3.0:1788204982.108635:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543496064, offset 0 00000400:00000200:1.0:1788204982.108639:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204982.108643:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:3.0:1788204982.108643:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00080000:1.0:1788204982.108646:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204982.108649:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108650:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204982.108652:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108653:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204982.108657:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:1.0:1788204982.108663:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204982.108666:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:3.0:1788204982.108667:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:1.0:1788204982.108668:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204982.108672:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788204982.108675:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788204982.108676:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:1.0:1788204982.108681:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788204982.108685:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000040:1.0:1788204982.108689:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108694:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204982.108694:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1788204982.108700:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204982.108702:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108705:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204982.108711:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788204982.108712:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471224:471224:256:4294967295] 192.168.201.118@tcp LPNI seq info [471083:471083:8:4294967295] 00000100:00080000:1.0:1788204982.108716:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108721:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.108723:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:1.0:1788204982.108724:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204982.108728:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204982.108730:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.108732:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204982.108734:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204982.108739:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.108742:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.108742:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000010:1.0:1788204982.108748:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788204982.108752:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000800:00000200:3.0:1788204982.108752:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:1.0:1788204982.108758:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204982.108760:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.108762:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:3.0:1788204982.108762:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 02000000:00000001:1.0:1788204982.108765:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204982.108767:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204982.108770:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb400. 00000800:00000200:3.0:1788204982.108771:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788204982.108775:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204982.108776:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00100000:1.0:1788204982.108782:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788204982.108783:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204982.108789:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:1.0:1788204982.108795:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1788204982.108796:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108799:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788204982.108800:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000100:00000040:1.0:1788204982.108804:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204982.108807:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1788204982.108813:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.108815:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1788204982.108817:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204982.108820:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788204982.108821:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000100:00000001:1.0:1788204982.108822:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204982.108824:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204982.108826:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204982.108828:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26668 00000400:00000010:3.0:1788204982.108832:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000400:00000200:1.0:1788204982.108835:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204982.108839:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204982.108840:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204982.108842:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204982.108842:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1788204982.108846:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204982.108847:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108850:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000200:3.0:1788204982.108852:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788204982.108853:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204982.108855:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788204982.108857:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204982.108860:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788204982.108863:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788204982.108867:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788204982.108872:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108873:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204982.108874:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788204982.108875:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108878:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204982.108881:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000040:3.0:1788204982.108882:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204982.108885:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204982.108890:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204982.108893:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204982.108895:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204982.108896:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204982.108899:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1788204982.108900:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.108902:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204982.108904:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788204982.108905:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.108907:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.108912:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204982.108913:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204982.108917:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000020:00000040:1.0:1788204982.108918:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788204982.108920:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204982.108921:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.108921:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321342213 00000100:00000001:1.0:1788204982.108926:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204982.108927:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204982.108928:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204982.108930:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcf00. 02000000:00000010:1.0:1788204982.108932:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba800. 00000100:00000001:1.0:1788204982.108935:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.108938:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00100000:1.0:1788204982.108939:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204982.108946:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204982.108947:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321342213 00000100:00000001:1.0:1788204982.108949:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204982.108952:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc660. 00000100:00000040:1.0:1788204982.108953:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.108956:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788204982.108961:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788204982.108964:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.108965:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204982.108970:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204982.108972:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788204982.108973:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788204982.108974:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1788204982.108977:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204982.108980:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1788204982.108981:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204982/real 1788204982] req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204982.108984:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26668 00000400:00000200:1.0:1788204982.108991:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204982.108996:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204982.108998:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204982.109000:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000001:3.0:1788204982.109000:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788204982.109001:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:3.0:1788204982.109003:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:1.0:1788204982.109005:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:3.0:1788204982.109007:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00000001:1.0:1788204982.109008:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204982.109012:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204982.109013:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788204982.109013:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204982.109015:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788204982.109016:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000200:3.0:1788204982.109016:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204982.109019:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000020:00000001:1.0:1788204982.109020:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788204982.109025:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204982.109025:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1788204982.109028:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1788204982.109031:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204982.109032:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204982.109036:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204982.109040:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788204982.109046:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204982.109049:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788204982.109050:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204982.109051:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204982.109054:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204982.109057:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.109061:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204982.109064:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788204982.109067:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.109070:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109072:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000010:1.0:1788204982.109074:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788204982.109076:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1788204982.109077:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788204982.109080:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1788204982.109081:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204982.109084:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109085:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204982.109086:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204982.109088:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204982.109124:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204982.109128:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000001:1.0:1788204982.109132:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204982.109136:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204982.109143:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204982.109146:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204982.109146:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204982.109154:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109159:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204982.109166:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204982.109169:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204982.109171:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26678) 00000100:00000001:3.0:1788204982.109178:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204982.109181:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109182:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.109182:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109184:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204982.109186:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204982.109188:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204982.109191:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204982.109193:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204982.109196:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204982.109201:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543496064:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788204982.109205:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109207:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.109208:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543496064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109212:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204982.109219:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204982.109222:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204982.109224:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204982.109226:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204982.109227:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204982.109229:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204982.109232:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:3.0:1788204982.109235:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204982.109236:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204982.109238:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788204982.109243:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204982.109244:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109245:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109252:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204982.109254:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109256:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204982.109257:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204982.109258:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204982.109260:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109265:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204982.109266:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.109267:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204982.109268:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204982.109270:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204982.109272:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496192:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204982.109274:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204982.109275:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204982.109276:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.109277:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204982.109279:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204982.109281:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.109283:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204982.109285:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204982.109288:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204982.109290:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543496192, portal 10 00000100:00000001:3.0:1788204982.109297:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204982.109298:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788204982.109302:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109307:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204982.109308:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204982.109310:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204982.109311:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543496192, offset 0 00000400:00000200:3.0:1788204982.109314:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204982.109316:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204982.109319:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204982.109321:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204982.109323:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204982.109324:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204982.109326:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204982.109328:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471225:471225:256:4294967295] 192.168.201.118@tcp LPNI seq info [471084:471084:8:4294967295] 00000400:00000200:3.0:1788204982.109331:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204982.109335:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204982.109338:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204982.109340:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204982.109342:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204982.109344:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204982.109348:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204982.109349:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1788204982.109351:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204982.109353:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204982.109354:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204982.109357:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204982.109359:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204982.109360:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204982.109361:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204982.109363:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109372:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.109374:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109378:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109379:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204982.109380:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204982.109383:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204982.109385:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204982.109386:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321340947 00000100:00000001:3.0:1788204982.109388:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321342213 00000100:00000001:3.0:1788204982.109389:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321340947 00000100:00000001:3.0:1788204982.109391:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321342213 00000100:00000001:3.0:1788204982.109392:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204982.109394:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109395:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204982.109397:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204982/real 1788204982] req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204982.109401:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204982.109402:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204982.109403:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204982.109404:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204982.109405:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204982.109407:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204982.109409:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204982.109417:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109423:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204982.109425:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109427:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.109428:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204982.109431:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109438:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204982.109442:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109448:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204982.109451:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109453:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.109454:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109456:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204982.109458:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204982.109461:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204982.109464:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204982.109467:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204982.109471:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204982.109477:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496192:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204982.109480:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204982.109482:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109486:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204982.109487:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.109489:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204982.109490:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204982.109491:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204982.109493:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496320:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204982.109495:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204982.109496:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204982.109498:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.109499:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204982.109502:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204982.109504:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.109507:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204982.109510:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204982.109513:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204982.109516:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543496320, portal 10 00000100:00000001:3.0:1788204982.109518:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204982.109519:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788204982.109525:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109532:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204982.109534:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204982.109536:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204982.109538:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543496320, offset 0 00000400:00000200:3.0:1788204982.109543:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204982.109546:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204982.109550:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204982.109553:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204982.109555:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204982.109557:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204982.109559:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204982.109562:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471226:471226:256:4294967295] 192.168.201.118@tcp LPNI seq info [471085:471085:8:4294967295] 00000400:00000200:3.0:1788204982.109567:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204982.109572:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204982.109577:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204982.109580:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204982.109584:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204982.109587:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204982.109590:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204982.109591:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1788204982.109593:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204982.109595:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204982.109597:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204982.109599:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204982.109601:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204982.109602:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204982.109604:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204982.109606:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109612:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.109614:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109618:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109619:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204982.109621:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204982.109623:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204982.109625:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204982.109627:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321340948 00000100:00000001:3.0:1788204982.109628:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321342213 00000100:00000001:3.0:1788204982.109630:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321340948 00000100:00000001:3.0:1788204982.109631:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321342213 00000100:00000001:3.0:1788204982.109633:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204982.109634:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109635:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204982.109637:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204982/real 1788204982] req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204982.109641:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204982.109642:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204982.109644:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204982.109645:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204982.109646:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204982.109648:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204982.109650:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204982.109658:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109664:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204982.109665:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109667:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.109668:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204982.109671:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109678:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204982.109682:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109690:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204982.109692:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109694:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.109695:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109697:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204982.109699:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204982.109701:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204982.109705:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204982.109707:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204982.109711:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204982.109725:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496320:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204982.109729:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204982.109732:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109736:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204982.109737:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.109739:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204982.109740:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204982.109742:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204982.109743:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496448:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204982.109745:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204982.109747:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204982.109748:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.109749:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204982.109751:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204982.109753:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.109755:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204982.109757:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204982.109759:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204982.109762:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543496448, portal 4 00000100:00000001:3.0:1788204982.109763:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204982.109764:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788204982.109768:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109772:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204982.109774:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204982.109775:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204982.109776:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543496448, offset 0 00000400:00000200:3.0:1788204982.109779:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204982.109782:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204982.109784:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204982.109786:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204982.109788:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204982.109790:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204982.109791:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204982.109793:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471227:471227:256:4294967295] 192.168.201.118@tcp LPNI seq info [471086:471086:8:4294967295] 00000400:00000200:3.0:1788204982.109796:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204982.109800:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204982.109803:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204982.109805:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204982.109807:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204982.109809:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204982.109812:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204982.109813:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1788204982.109816:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204982.109817:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204982.109819:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204982.109821:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204982.109823:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204982.109824:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204982.109825:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204982.109827:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109832:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.109834:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109838:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109840:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204982.109842:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204982.109845:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204982.109848:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204982.109851:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321340957 00000100:00000001:3.0:1788204982.109853:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321342213 00000100:00000001:3.0:1788204982.109855:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321340957 00000100:00000001:3.0:1788204982.109857:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321342213 00000100:00000001:3.0:1788204982.109859:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204982.109861:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.109863:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204982.109867:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204982/real 1788204982] req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204982.109875:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204982.109876:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204982.109878:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204982.109880:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204982.109882:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204982.109884:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204982.109887:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204982.109891:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109896:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204982.109897:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109899:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.109900:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204982.109903:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109910:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204982.109914:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109920:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204982.109923:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109925:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.109926:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.109927:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204982.109929:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204982.109932:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204982.109935:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204982.109937:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204982.109940:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204982.109948:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496448:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204982.109952:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204982.109954:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109958:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204982.109959:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.109961:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204982.109962:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204982.109963:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204982.109964:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496576:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204982.109966:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204982.109968:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204982.109969:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204982.109970:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204982.109973:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204982.109974:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204982.109976:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204982.109978:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204982.109980:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204982.109982:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543496576, portal 4 00000100:00000001:3.0:1788204982.109984:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204982.109985:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788204982.109989:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.109993:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204982.109994:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204982.109996:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204982.109997:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543496576, offset 0 00000400:00000200:3.0:1788204982.109999:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204982.110002:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204982.110004:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204982.110006:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204982.110007:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204982.110009:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204982.110010:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204982.110012:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471228:471228:256:4294967295] 192.168.201.118@tcp LPNI seq info [471087:471087:8:4294967295] 00000400:00000200:3.0:1788204982.110015:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204982.110019:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204982.110021:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204982.110023:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204982.110025:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204982.110027:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:3.1:1788204982.110029:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204982.110031:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:3.0:1788204982.110033:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204982.110034:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204982.110036:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204982.110038:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204982.110040:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204982.110041:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204982.110042:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204982.110044:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110052:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.110054:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205007 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110058:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.110059:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204982.110061:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204982.110062:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204982.110064:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204982.110065:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321340980 00000100:00000001:3.0:1788204982.110066:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321342213 00000100:00000001:3.0:1788204982.110068:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321340980 00000100:00000001:3.0:1788204982.110070:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321342213 00000100:00000001:3.0:1788204982.110072:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204982.110074:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.110076:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204982.110079:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204982/real 1788204982] req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204982.110086:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204982.110088:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204982.110116:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204982.110119:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204982.110121:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204982.110123:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204982.110126:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204982.110131:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110136:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204982.110137:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110138:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.110139:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204982.110142:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110146:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204982.110148:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110152:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204982.110153:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110154:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204982.110155:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110156:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204982.110157:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204982.110158:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204982.110160:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204982.110161:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204982.110163:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204982.110167:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496576:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204982.110170:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110171:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.110173:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543496192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110177:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204982.110178:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204982.110179:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204982.110181:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204982.110181:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204982.110183:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204982.110184:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204982.110185:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:3.0:1788204982.110187:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204982.110188:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204982.110189:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788204982.110193:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204982.110194:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110195:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.110197:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543496320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110200:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204982.110201:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204982.110202:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204982.110203:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204982.110204:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204982.110204:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204982.110205:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204982.110206:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:3.0:1788204982.110207:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204982.110208:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204982.110209:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788204982.110210:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204982.110210:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110211:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.110213:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543496448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110216:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204982.110216:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204982.110217:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204982.110218:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204982.110219:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204982.110219:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204982.110220:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204982.110221:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:3.0:1788204982.110222:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204982.110223:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204982.110224:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788204982.110226:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204982.110227:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110227:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204982.110229:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543496576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205007 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204982.110232:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204982.110233:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204982.110234:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204982.110235:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204982.110235:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204982.110236:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204982.110237:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204982.110237:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:3.0:1788204982.110238:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204982.110239:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204982.110240:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788204982.110241:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204982.110241:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110242:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204982.110245:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204982.110245:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.110246:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204982.110247:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.940147:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204982.940156:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204982.940159:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204982.940165:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788204982.940170:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204982.940172:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204982.940174:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204982.940174:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204982.940176:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.940181:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204982.940181:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204982.940183:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204982.940185:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204982.940186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.940188:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788204982.940189:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204982.940190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204982.940193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204982.940193:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.940194:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204982.940196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204982.940198:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204982.940199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204982.940203:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204982.940206:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204982.940208:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204982.940233:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204982.940236:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204982.940240:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204983.452165:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321342213, at 4321342272 00000100:00000040:3.0:1788204983.452165:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321342213, at 4321342272 00000100:00000010:2.0:1788204983.452183:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf00. 00000100:00000010:3.0:1788204983.452189:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc660. 00000100:00000001:1.0:1788204983.964218:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204983.964218:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204983.964228:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204983.964228:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204983.964231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204983.964231:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204983.964232:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204983.964239:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204983.964240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204983.964246:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204983.964247:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204983.964248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204983.964251:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204983.964252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204983.964254:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204983.964256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204983.964256:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204983.964260:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204983.964260:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204983.964261:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204983.964264:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204983.964267:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204983.964268:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964269:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204983.964273:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964277:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204983.964281:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204983.964282:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204983.964284:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964285:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204983.964285:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964287:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964288:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204983.964292:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204983.964293:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204983.964295:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964296:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204983.964297:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964298:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204983.964299:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204983.964350:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204983.964354:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204983.964356:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204984.988220:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204984.988220:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204984.988227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204984.988230:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204984.988230:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204984.988235:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204984.988235:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204984.988236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204984.988243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204984.988243:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204984.988247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204984.988249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204984.988249:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204984.988251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204984.988255:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204984.988255:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204984.988258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204984.988260:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204984.988261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204984.988265:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204984.988267:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204984.988269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204984.988270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204984.988271:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204984.988274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204984.988275:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204984.988347:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204984.988352:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204984.988356:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204986.012197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204986.012197:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204986.012207:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204986.012209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204986.012210:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204986.012211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204986.012214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204986.012216:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204986.012218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204986.012220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204986.012221:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204986.012227:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204986.012227:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204986.012228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204986.012231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204986.012232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204986.012233:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204986.012237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204986.012238:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204986.012241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204986.012243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204986.012243:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204986.012245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204986.012250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204986.012254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204986.012258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204986.012279:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204986.012282:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204986.012285:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204987.036193:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204987.036193:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204987.036199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204987.036200:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204987.036201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204987.036203:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204987.036204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204987.036205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204987.036210:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788204987.036212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204987.036214:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204987.036215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204987.036217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204987.036218:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204987.036219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204987.036221:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.036222:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204987.036225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204987.036226:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204987.036229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204987.036233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204987.036234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204987.036236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204987.036242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204987.036243:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204987.036246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204987.036271:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204987.036273:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204987.036276:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204987.228225:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204987.228236:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204987.228239:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204987.228243:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204987.228248:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204987.228255:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26673 00000400:00000200:2.0:1788204987.228264:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204987.228270:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204987.228274:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204987.228278:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228281:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204987.228283:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228285:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204987.228288:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204987.228289:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204987.228292:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204987.228294:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204987.228295:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204987.228297:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204987.228299:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228300:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204987.228301:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228302:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204987.228311:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204987.228316:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228318:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204987.228322:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204987.228325:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204987.228327:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228328:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204987.228330:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204987.228337:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788204987.228340:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204987.228342:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204987.228343:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228346:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204987.228348:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204987.228351:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204987.228358:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788204987.228364:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204987.228372:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204987.228382:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204987.228386:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204987.228391:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204987.228410:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204987.228412:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204987.228414:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204987.228415:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204987.228416:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204987.228419:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204987.228422:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26673 00000400:00000200:2.0:1788204987.228426:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204987.228430:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204987.228433:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204987.228437:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228439:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204987.228441:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228444:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204987.228446:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204987.228448:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:3.0:1788204987.228448:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788204987.228450:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204987.228454:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:3.0:1788204987.228455:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788204987.228457:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788204987.228459:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788204987.228461:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788204987.228467:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204987.228468:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228472:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204987.228475:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228479:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204987.228483:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:3.0:1788204987.228483:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204987.228497:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204987.228498:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228501:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.228502:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788204987.228503:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788204987.228505:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204987.228508:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204987.228510:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204987.228516:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228518:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204987.228521:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788204987.228523:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228525:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204987.228527:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543496704:192.168.201.118@tcp:250:kworker.0 00000100:00000010:2.0:1788204987.228528:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000020:00000040:2.0:1788204987.228531:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204987.228533:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204987.228534:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228538:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204987.228540:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204987.228540:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204987.228543:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204987.228547:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788204987.228547:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204987.228551:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228552:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.228558:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788204987.228560:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204987.228566:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:3.0:1788204987.228566:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:2.0:1788204987.228569:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.228571:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204987.228573:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204987.228581:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204987.228581:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:2.0:1788204987.228586:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204987.228587:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00080000:2.0:1788204987.228589:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788204987.228591:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204987.228592:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204987.228594:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:3.0:1788204987.228594:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00080000:2.0:1788204987.228598:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26673 00000400:00000200:2.0:1788204987.228601:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:3.0:1788204987.228602:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543496704, portal 25 00000400:00000200:2.0:1788204987.228605:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204987.228606:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788204987.228608:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204987.228609:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922048 : -109424377629568 : ffff9c7aa6d77480) 00000100:00000040:2.0:1788204987.228611:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228615:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204987.228616:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228618:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204987.228621:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000040:3.0:1788204987.228621:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204987.228622:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788204987.228624:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204987.228626:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788204987.228629:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204987.228633:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204987.228639:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204987.228641:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788204987.228642:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204987.228648:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:3.0:1788204987.228650:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228651:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204987.228654:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204987.228656:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:3.0:1788204987.228657:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000001:2.0:1788204987.228658:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204987.228661:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:3.0:1788204987.228663:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543496704, offset 0 02000000:00000001:2.0:1788204987.228664:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204987.228669:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204987.228674:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228677:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204987.228681:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788204987.228683:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00000001:2.0:1788204987.228684:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228688:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204987.228691:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000400:00000200:3.0:1788204987.228693:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000040:2.0:1788204987.228694:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788204987.228697:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204987.228699:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204987.228699:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788204987.228702:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204987.228704:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204987.228704:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000001:2.0:1788204987.228706:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204987.228709:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000400:00000200:3.0:1788204987.228710:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1788204987.228714:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204987.228714:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00100000:2.0:1788204987.228721:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788204987.228722:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471229:471229:256:4294967295] 192.168.201.118@tcp LPNI seq info [471088:471088:8:4294967295] 00000400:00000200:3.0:1788204987.228732:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788204987.228737:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204987.228739:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204987.228746:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204987.228747:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:2.0:1788204987.228759:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204987.228759:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788204987.228763:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:3.0:1788204987.228764:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000100:00080000:2.0:1788204987.228765:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788204987.228768:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204987.228770:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204987.228773:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.0:1788204987.228774:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788204987.228777:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26673 00000800:00000200:3.0:1788204987.228778:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000400:00000200:2.0:1788204987.228783:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.1:1788204987.228787:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:2.0:1788204987.228788:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204987.228791:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:3.1:1788204987.228793:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00000040:2.0:1788204987.228796:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228798:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1788204987.228800:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000040:2.0:1788204987.228803:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000010:3.0:1788204987.228805:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000100:00000001:2.0:1788204987.228806:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204987.228808:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204987.228809:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204987.228810:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1788204987.228811:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:2.0:1788204987.228812:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204987.228815:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788204987.228817:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1788204987.228818:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788204987.228822:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204987.228822:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000100:00000001:2.0:1788204987.228826:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1788204987.228826:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000040:2.0:1788204987.228828:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.228830:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788204987.228831:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:2.0:1788204987.228832:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204987.228836:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000200:3.0:1788204987.228838:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204987.228839:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204987.228842:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204987.228845:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204987.228848:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228852:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204987.228854:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:3.0:1788204987.228855:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788204987.228857:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.228860:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204987.228860:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204987.228865:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788204987.228869:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788204987.228871:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204987.228872:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204987.228874:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.228875:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204987.228877:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204987.228881:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204987.228884:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204987.228886:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788204987.228889:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.228890:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321347333 02000000:00000010:2.0:1788204987.228892:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788204987.228896:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204987.228896:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc660. 00000100:00100000:2.0:1788204987.228903:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204987.228904:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1788204987.228910:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204987.228912:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.228912:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321347333 00000100:00000010:3.0:1788204987.228916:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fccc0. 00000100:00000040:2.0:1788204987.228918:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.228920:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788204987.228924:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788204987.228927:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.228928:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204987.228930:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204987.228934:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204987.228934:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788204987.228938:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204987.228940:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788204987.228944:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204987/real 1788204987] req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204987.228945:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204987.228951:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26673 00000400:00000200:2.0:1788204987.228959:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000001:3.0:1788204987.228964:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1788204987.228965:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000001:3.0:1788204987.228967:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1788204987.228969:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000010:3.0:1788204987.228971:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00000040:2.0:1788204987.228974:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204987.228977:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000100:00000001:2.0:1788204987.228978:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1788204987.228980:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000040:2.0:1788204987.228983:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204987.228985:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788204987.228987:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204987.228992:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000200:3.0:1788204987.228992:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204987.228994:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204987.228999:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:3.0:1788204987.229016:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229023:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1788204987.229026:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:3.0:1788204987.229027:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229030:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1788204987.229032:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788204987.229033:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788204987.229036:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00100000:3.0:1788204987.229040:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204987.229044:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204987.229050:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204987.229054:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204987.229056:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204987.229058:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204987.229064:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000040:3.0:1788204987.229065:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788204987.229071:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204987.229077:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229081:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788204987.229082:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204987.229086:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788204987.229087:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204987.229109:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.229111:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204987.229112:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788204987.229114:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.229115:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204987.229118:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788204987.229120:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204987.229122:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204987.229123:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204987.229125:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204987.229126:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204987.229127:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229130:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:2.0:1788204987.229131:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:3.0:1788204987.229134:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204987.229137:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229138:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204987.229142:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:2.0:1788204987.229145:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788204987.229147:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204987.229156:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204987.229160:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788204987.229162:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204987.229166:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204987.229168:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204987.229174:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204987.229184:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543496704:192.168.201.118@tcp:250:kworker.0 00000100:00000001:2.0:1788204987.229189:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229192:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229195:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788204987.229196:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26683) 00000100:00000040:3.0:1788204987.229200:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77480 x1875055543496704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229208:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204987.229211:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204987.229214:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204987.229217:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204987.229219:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204987.229220:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204987.229222:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204987.229225:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067ec00. 02000000:00000001:3.0:1788204987.229229:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204987.229231:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204987.229233:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77480. 00000100:00000001:3.0:1788204987.229236:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204987.229237:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229239:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229246:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204987.229248:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229250:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204987.229252:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204987.229253:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204987.229257:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229263:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204987.229265:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.229267:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204987.229269:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204987.229271:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204987.229274:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496832:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204987.229277:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204987.229279:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204987.229280:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.229282:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204987.229285:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204987.229287:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204987.229290:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204987.229293:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204987.229297:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204987.229300:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543496832, portal 10 00000100:00000001:3.0:1788204987.229302:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204987.229303:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331915776 : -109424377635840 : ffff9c7aa6d75c00) 00000100:00000040:3.0:1788204987.229309:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229316:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204987.229318:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204987.229320:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204987.229322:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543496832, offset 0 00000400:00000200:3.0:1788204987.229326:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204987.229330:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204987.229334:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204987.229338:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204987.229340:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204987.229342:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204987.229345:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204987.229348:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471230:471230:256:4294967295] 192.168.201.118@tcp LPNI seq info [471089:471089:8:4294967295] 00000400:00000200:3.0:1788204987.229352:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204987.229359:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204987.229364:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204987.229367:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204987.229371:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204987.229374:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204987.229378:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204987.229381:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788204987.229385:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204987.229387:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204987.229389:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204987.229392:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204987.229395:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204987.229397:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204987.229399:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204987.229403:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229410:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204987.229413:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229439:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229440:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204987.229444:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204987.229449:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204987.229452:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204987.229454:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321346067 00000100:00000001:3.0:1788204987.229457:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321347333 00000100:00000001:3.0:1788204987.229460:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321346067 00000100:00000001:3.0:1788204987.229462:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321347333 00000100:00000001:3.0:1788204987.229464:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204987.229466:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229468:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204987.229471:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204987/real 1788204987] req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204987.229478:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204987.229479:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204987.229481:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204987.229483:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204987.229485:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204987.229487:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204987.229490:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204987.229498:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229503:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204987.229504:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229506:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204987.229507:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204987.229510:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229516:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204987.229519:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229525:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204987.229527:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229529:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204987.229530:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229532:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204987.229534:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204987.229536:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204987.229539:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204987.229541:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204987.229544:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204987.229551:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496832:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204987.229554:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204987.229557:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229563:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204987.229564:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.229566:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204987.229568:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204987.229569:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204987.229571:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496960:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204987.229574:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204987.229576:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204987.229577:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.229579:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204987.229582:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204987.229583:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204987.229586:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204987.229588:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204987.229591:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204987.229593:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543496960, portal 10 00000100:00000001:3.0:1788204987.229595:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204987.229596:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331918464 : -109424377633152 : ffff9c7aa6d76680) 00000100:00000040:3.0:1788204987.229600:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229606:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204987.229609:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204987.229611:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204987.229613:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543496960, offset 0 00000400:00000200:3.0:1788204987.229616:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204987.229619:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204987.229623:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204987.229625:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204987.229628:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204987.229630:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204987.229632:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204987.229635:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471231:471231:256:4294967295] 192.168.201.118@tcp LPNI seq info [471090:471090:8:4294967295] 00000400:00000200:3.0:1788204987.229639:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204987.229644:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204987.229648:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204987.229651:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204987.229655:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204987.229658:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204987.229662:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204987.229664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788204987.229668:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204987.229669:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204987.229672:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204987.229674:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204987.229676:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204987.229677:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204987.229679:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204987.229681:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229689:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204987.229690:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229694:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229695:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204987.229697:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204987.229698:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204987.229700:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204987.229701:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321346068 00000100:00000001:3.0:1788204987.229703:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321347333 00000100:00000001:3.0:1788204987.229704:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321346068 00000100:00000001:3.0:1788204987.229705:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321347333 00000100:00000001:3.0:1788204987.229707:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204987.229708:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229709:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204987.229711:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204987/real 1788204987] req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204987.229715:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204987.229716:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204987.229716:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204987.229718:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204987.229719:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204987.229720:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204987.229722:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204987.229726:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229729:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204987.229731:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229732:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204987.229733:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204987.229734:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229738:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204987.229740:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229744:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204987.229745:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229746:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204987.229746:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229747:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204987.229749:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204987.229750:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204987.229752:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204987.229753:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204987.229755:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204987.229759:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543496960:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204987.229761:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204987.229763:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229766:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204987.229767:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.229769:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204987.229770:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204987.229771:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204987.229773:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497088:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204987.229775:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204987.229776:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204987.229777:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.229778:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204987.229780:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204987.229781:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204987.229783:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204987.229784:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204987.229786:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204987.229788:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543497088, portal 4 00000100:00000001:3.0:1788204987.229789:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204987.229789:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331914880 : -109424377636736 : ffff9c7aa6d75880) 00000100:00000040:3.0:1788204987.229793:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229796:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204987.229797:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204987.229799:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204987.229800:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543497088, offset 0 00000400:00000200:3.0:1788204987.229802:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204987.229804:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204987.229806:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204987.229807:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204987.229809:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204987.229810:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204987.229811:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204987.229813:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471232:471232:256:4294967295] 192.168.201.118@tcp LPNI seq info [471091:471091:8:4294967295] 00000400:00000200:3.0:1788204987.229815:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204987.229819:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204987.229822:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204987.229823:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204987.229826:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204987.229827:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204987.229829:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204987.229831:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788204987.229833:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204987.229835:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204987.229836:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204987.229838:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204987.229840:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204987.229841:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204987.229842:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204987.229843:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229848:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204987.229850:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229853:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229854:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204987.229855:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204987.229856:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204987.229858:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204987.229859:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321346077 00000100:00000001:3.0:1788204987.229860:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321347333 00000100:00000001:3.0:1788204987.229862:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321346077 00000100:00000001:3.0:1788204987.229863:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321347333 00000100:00000001:3.0:1788204987.229864:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204987.229865:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.229866:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204987.229868:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204987/real 1788204987] req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204987.229872:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204987.229873:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204987.229874:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204987.229875:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204987.229876:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204987.229878:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204987.229879:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204987.229884:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229889:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204987.229890:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229892:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204987.229894:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204987.229897:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229903:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204987.229906:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229912:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204987.229915:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229916:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204987.229918:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.229919:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204987.229921:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204987.229923:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204987.229926:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204987.229928:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204987.229932:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204987.229943:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497088:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204987.229947:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204987.229950:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.229956:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204987.229957:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.229958:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204987.229960:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204987.229962:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204987.229964:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497216:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204987.229967:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204987.229969:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204987.229970:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204987.229972:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204987.229974:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204987.229976:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204987.229979:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204987.229981:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204987.229984:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204987.229987:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543497216, portal 4 00000100:00000001:3.0:1788204987.229989:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204987.229990:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331919360 : -109424377632256 : ffff9c7aa6d76a00) 00000100:00000040:3.0:1788204987.229995:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230016:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204987.230018:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204987.230020:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204987.230023:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543497216, offset 0 00000400:00000200:3.0:1788204987.230027:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204987.230030:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204987.230034:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204987.230036:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204987.230038:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204987.230041:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204987.230043:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204987.230046:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471233:471233:256:4294967295] 192.168.201.118@tcp LPNI seq info [471092:471092:8:4294967295] 00000400:00000200:3.0:1788204987.230051:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204987.230057:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204987.230061:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204987.230064:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204987.230068:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204987.230071:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788204987.230075:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204987.230078:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788204987.230081:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204987.230083:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204987.230086:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204987.230106:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204987.230109:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204987.230111:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204987.230114:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204987.230118:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230125:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204987.230128:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205012 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230135:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.230136:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204987.230139:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204987.230143:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204987.230146:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204987.230148:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321346101 00000100:00000001:3.0:1788204987.230150:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321347333 00000100:00000001:3.0:1788204987.230152:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321346101 00000100:00000001:3.0:1788204987.230154:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321347333 00000100:00000001:3.0:1788204987.230157:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204987.230159:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.230161:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204987.230164:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204987/real 1788204987] req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204987.230171:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204987.230173:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204987.230174:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204987.230176:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204987.230178:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204987.230180:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204987.230183:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204987.230192:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230199:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204987.230200:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230202:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204987.230203:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204987.230207:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230213:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204987.230217:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230224:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204987.230227:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230228:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204987.230229:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230231:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204987.230233:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204987.230236:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204987.230239:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204987.230241:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204987.230245:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204987.230253:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497216:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204987.230256:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230258:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204987.230261:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75c00 x1875055543496832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230268:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204987.230270:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204987.230272:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204987.230274:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204987.230275:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204987.230277:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204987.230279:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204987.230281:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788204987.230283:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204987.230285:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204987.230286:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75c00. 00000100:00000001:3.0:1788204987.230289:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204987.230290:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230292:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204987.230295:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76680 x1875055543496960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230307:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204987.230309:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204987.230310:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204987.230312:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204987.230314:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204987.230315:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204987.230317:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204987.230318:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:3.0:1788204987.230321:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204987.230322:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204987.230323:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76680. 00000100:00000001:3.0:1788204987.230325:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204987.230327:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230328:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204987.230331:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75880 x1875055543497088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230338:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204987.230339:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204987.230341:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204987.230343:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204987.230344:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204987.230345:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204987.230347:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204987.230348:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:3.0:1788204987.230350:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204987.230351:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204987.230353:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75880. 00000100:00000001:3.0:1788204987.230355:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204987.230356:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230358:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204987.230361:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76a00 x1875055543497216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205012 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204987.230367:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204987.230368:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204987.230370:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204987.230372:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204987.230373:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204987.230374:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204987.230376:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204987.230377:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:3.0:1788204987.230379:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204987.230380:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204987.230381:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76a00. 00000100:00000001:3.0:1788204987.230384:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204987.230385:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230387:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204987.230391:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204987.230392:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204987.230394:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204987.230396:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204988.060185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204988.060185:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204988.060190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204988.060194:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204988.060195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204988.060196:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204988.060198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204988.060204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204988.060205:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204988.060211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204988.060211:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204988.060217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204988.060217:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204988.060222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204988.060223:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204988.060227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204988.060227:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204988.060233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204988.060234:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204988.060237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204988.060243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204988.060244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204988.060247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204988.060248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204988.060251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204988.060253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204988.060301:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204988.060305:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204988.060307:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204988.572253:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321347333, at 4321347392 00000100:00000040:2.0:1788204988.572254:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321347333, at 4321347392 00000100:00000010:3.0:1788204988.572277:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fccc0. 00000100:00000010:2.0:1788204988.572285:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc660. 00000100:00000001:3.0:1788204989.084231:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204989.084232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204989.084241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204989.084248:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204989.084249:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204989.084249:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204989.084258:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788204989.084258:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204989.084259:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204989.084265:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204989.084274:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204989.084276:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:1.0:1788204989.084276:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204989.084282:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204989.084289:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204989.084291:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204989.084291:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204989.084293:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204989.084297:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204989.084301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204989.084301:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204989.084305:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204989.084305:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204989.084306:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204989.084313:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204989.084315:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204989.084316:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204989.084318:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204989.084321:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204989.084324:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204989.084329:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204989.084332:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788204989.084360:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788204989.084363:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788204989.084365:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204990.108188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204990.108188:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204990.108192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204990.108198:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204990.108200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204990.108200:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204990.108203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204990.108204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.108204:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204990.108206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204990.108209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204990.108211:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204990.108212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204990.108214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204990.108215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.108216:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204990.108218:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.108220:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204990.108221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204990.108221:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204990.108224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204990.108226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204990.108227:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204990.108231:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204990.108232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204990.108234:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204990.108286:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204990.108290:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204990.108297:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204991.132206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204991.132206:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204991.132211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204991.132217:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204991.132217:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204991.132218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204991.132221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204991.132222:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204991.132228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788204991.132229:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204991.132229:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204991.132230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204991.132236:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204991.132237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204991.132237:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204991.132241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204991.132242:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204991.132242:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204991.132244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204991.132246:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204991.132248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204991.132249:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204991.132250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204991.132254:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204991.132255:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204991.132259:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204991.132283:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204991.132291:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204991.132293:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204992.156177:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204992.156181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204992.156181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204992.156187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204992.156189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204992.156189:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204992.156193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.156193:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204992.156194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204992.156197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204992.156200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204992.156201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204992.156203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204992.156205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204992.156206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204992.156208:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204992.156209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204992.156209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.156213:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204992.156215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204992.156216:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204992.156218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.156220:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204992.156222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.156223:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.156226:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204992.156272:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204992.156276:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204992.156280:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788204992.348173:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204992.348183:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788204992.348186:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204992.348189:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204992.348193:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204992.348200:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26678 00000400:00000200:2.0:1788204992.348211:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204992.348218:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204992.348222:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204992.348225:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348229:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204992.348231:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348234:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204992.348238:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204992.348241:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788204992.348245:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204992.348248:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788204992.348251:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788204992.348254:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788204992.348256:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348258:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204992.348260:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348262:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204992.348265:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788204992.348268:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348271:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204992.348274:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204992.348278:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204992.348281:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.348284:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788204992.348287:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788204992.348298:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788204992.348302:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788204992.348307:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204992.348309:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.348312:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204992.348315:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204992.348320:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204992.348328:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788204992.348336:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204992.348345:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788204992.348356:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204992.348361:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204992.348366:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204992.348394:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204992.348397:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204992.348401:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788204992.348402:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204992.348404:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204992.348407:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204992.348411:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26678 00000400:00000200:2.0:1788204992.348415:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204992.348419:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204992.348421:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204992.348422:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788204992.348425:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348429:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204992.348430:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788204992.348432:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204992.348434:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788204992.348436:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204992.348440:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204992.348441:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788204992.348444:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:3.0:1788204992.348445:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204992.348447:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788204992.348451:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788204992.348455:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788204992.348461:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204992.348465:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788204992.348466:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204992.348469:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204992.348470:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.348475:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788204992.348480:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:3.0:1788204992.348480:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1788204992.348484:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:3.0:1788204992.348486:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204992.348489:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348493:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204992.348494:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543497344:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788204992.348497:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788204992.348502:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788204992.348504:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204992.348507:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.348508:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788204992.348510:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788204992.348512:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788204992.348515:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788204992.348517:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788204992.348520:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.348523:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204992.348524:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000010:2.0:1788204992.348527:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 02000000:00000001:3.0:1788204992.348528:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788204992.348533:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000400:00000200:3.0:1788204992.348538:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788204992.348543:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204992.348545:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788204992.348545:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:2.0:1788204992.348548:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204992.348550:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204992.348551:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204992.348557:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000400:00000010:3.0:1788204992.348557:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000001:2.0:1788204992.348560:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204992.348565:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788204992.348569:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543497344, portal 25 00000100:00000001:2.0:1788204992.348571:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204992.348574:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.348574:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204992.348578:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 00000100:00000040:2.0:1788204992.348581:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788204992.348588:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204992.348594:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788204992.348600:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788204992.348602:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788204992.348606:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:3.0:1788204992.348606:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348610:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:3.0:1788204992.348612:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00080000:2.0:1788204992.348613:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1788204992.348615:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543497344, offset 0 00000100:00000001:2.0:1788204992.348619:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204992.348624:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26678 00000400:00000200:3.0:1788204992.348627:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:2.0:1788204992.348632:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204992.348637:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:3.0:1788204992.348639:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:2.0:1788204992.348641:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204992.348646:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204992.348648:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:2.0:1788204992.348650:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788204992.348654:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788204992.348655:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348659:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204992.348660:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000020:00000001:2.0:1788204992.348664:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000200:3.0:1788204992.348666:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1788204992.348667:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:3.0:1788204992.348670:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000001:2.0:1788204992.348672:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1788204992.348675:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471234:471234:256:4294967295] 192.168.201.118@tcp LPNI seq info [471093:471093:8:4294967295] 00000020:00000040:2.0:1788204992.348676:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000200:3.0:1788204992.348681:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:2.0:1788204992.348682:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788204992.348686:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788204992.348693:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348695:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204992.348703:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348707:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204992.348710:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000020:00000040:2.0:1788204992.348711:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788204992.348718:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000200:3.0:1788204992.348719:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348723:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204992.348726:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:3.0:1788204992.348727:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 02000000:00000001:2.0:1788204992.348729:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204992.348732:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.348736:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000200:3.0:1788204992.348737:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000008:00100000:2.0:1788204992.348740:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000800:00000200:3.0:1788204992.348741:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000008:00000001:2.0:1788204992.348745:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.348748:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788204992.348749:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:2.0:1788204992.348753:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000800:00000200:3.1:1788204992.348753:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000020:00000040:2.0:1788204992.348759:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000800:00000100:3.0:1788204992.348760:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000001:2.0:1788204992.348764:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000800:00000010:3.0:1788204992.348765:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 02000000:00000001:2.0:1788204992.348767:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.348771:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204992.348773:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204992.348773:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1788204992.348777:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204992.348779:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 02000000:00000010:2.0:1788204992.348785:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000400:00000200:3.0:1788204992.348785:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204992.348787:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204992.348791:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204992.348795:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204992.348799:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.348805:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1788204992.348810:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204992.348811:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.348835:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.348836:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788204992.348838:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788204992.348841:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.348844:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788204992.348847:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204992.348851:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204992.348856:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321352453 00000100:00000001:2.0:1788204992.348862:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788204992.348863:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fccc0. 00000100:00080000:2.0:1788204992.348867:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204992.348871:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788204992.348871:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1788204992.348872:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204992.348875:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204992.348878:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788204992.348879:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321352453 00000100:00080000:2.0:1788204992.348882:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26678 00000100:00000010:3.0:1788204992.348884:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcd80. 00000100:00000001:3.0:1788204992.348888:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000400:00000200:2.0:1788204992.348890:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204992.348894:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204992.348898:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788204992.348898:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788204992.348902:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204992.348903:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.348907:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204992.348908:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1788204992.348921:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.348926:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00100000:3.0:1788204992.348926:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204992/real 1788204992] req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788204992.348930:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204992.348933:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788204992.348936:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788204992.348940:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788204992.348944:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788204992.348948:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000001:3.0:1788204992.348951:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1788204992.348954:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000001:3.0:1788204992.348954:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788204992.348958:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:3.0:1788204992.348958:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204992.348961:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000100:00000040:2.0:1788204992.348964:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:3.0:1788204992.348964:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204992.348968:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788204992.348969:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000200:3.0:1788204992.348973:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788204992.348974:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788204992.348982:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000200:3.0:1788204992.348985:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204992.348986:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788204992.348990:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204992.348993:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204992.348994:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788204992.348996:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.348997:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204992.348998:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204992.349001:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:3.0:1788204992.349002:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:2.0:1788204992.349003:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.349007:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349007:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1788204992.349010:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788204992.349014:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00100000:3.0:1788204992.349014:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788204992.349018:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204992.349020:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.349022:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204992.349024:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204992.349026:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204992.349030:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:3.0:1788204992.349032:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788204992.349034:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788204992.349039:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788204992.349042:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788204992.349052:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204992.349056:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349058:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1788204992.349061:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349062:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349066:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.349068:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349072:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788204992.349074:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349076:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788204992.349077:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788204992.349082:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788204992.349082:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788204992.349083:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788204992.349087:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788204992.349089:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788204992.349107:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26678 00000400:00000200:2.0:1788204992.349110:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788204992.349113:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788204992.349114:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788204992.349116:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.349118:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204992.349119:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00080000:3.0:1788204992.349121:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:2.0:1788204992.349122:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788204992.349125:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788204992.349127:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788204992.349129:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788204992.349129:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:2.0:1788204992.349131:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788204992.349134:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788204992.349136:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788204992.349139:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788204992.349140:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.349144:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788204992.349147:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788204992.349150:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788204992.349154:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788204992.349159:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788204992.349162:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204992.349163:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543497344:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788204992.349164:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788204992.349166:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788204992.349171:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.349172:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788204992.349173:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1788204992.349173:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349176:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788204992.349177:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788204992.349181:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204992.349181:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543497344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788204992.349186:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788204992.349189:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788204992.349193:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788204992.349194:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349194:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788204992.349197:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788204992.349198:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788204992.349199:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788204992.349201:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788204992.349203:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788204992.349204:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204992.349206:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 00000100:00100000:2.0:1788204992.349210:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788204992.349210:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204992.349213:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204992.349216:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204992.349220:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000100:00000001:2.0:1788204992.349221:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788204992.349223:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204992.349223:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d800. 00000100:00000040:2.0:1788204992.349229:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204992.349230:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204992.349233:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204992.349235:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:2.0:1788204992.349240:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349241:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204992.349243:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349245:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788204992.349246:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26688) 00000100:00000001:3.0:1788204992.349255:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204992.349257:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349259:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204992.349261:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204992.349262:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204992.349267:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349275:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204992.349276:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.349278:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204992.349280:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204992.349282:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204992.349285:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497472:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204992.349289:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204992.349291:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204992.349293:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.349295:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204992.349298:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204992.349301:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204992.349304:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204992.349307:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204992.349311:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204992.349314:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543497472, portal 10 00000100:00000001:3.0:1788204992.349315:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204992.349316:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000040:3.0:1788204992.349321:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349326:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204992.349327:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204992.349329:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204992.349331:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543497472, offset 0 00000400:00000200:3.0:1788204992.349335:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204992.349339:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204992.349343:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204992.349346:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204992.349348:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204992.349351:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204992.349353:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204992.349356:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471235:471235:256:4294967295] 192.168.201.118@tcp LPNI seq info [471094:471094:8:4294967295] 00000400:00000200:3.0:1788204992.349361:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204992.349368:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204992.349372:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204992.349376:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204992.349379:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204992.349381:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204992.349384:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204992.349386:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788204992.349388:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204992.349390:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204992.349392:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204992.349394:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204992.349396:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204992.349398:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204992.349399:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204992.349402:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349407:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204992.349409:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349413:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349414:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204992.349416:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204992.349419:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204992.349421:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204992.349423:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321351187 00000100:00000001:3.0:1788204992.349425:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321352453 00000100:00000001:3.0:1788204992.349426:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321351187 00000100:00000001:3.0:1788204992.349428:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321352453 00000100:00000001:3.0:1788204992.349429:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204992.349430:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349432:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204992.349434:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204992/real 1788204992] req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204992.349438:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204992.349439:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204992.349440:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204992.349442:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204992.349443:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204992.349444:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204992.349446:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204992.349450:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349454:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204992.349454:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349455:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.349456:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204992.349458:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349463:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204992.349465:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349468:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204992.349469:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349470:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.349471:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349472:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204992.349473:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204992.349475:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204992.349477:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204992.349478:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204992.349480:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204992.349484:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497472:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204992.349487:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204992.349489:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349493:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204992.349494:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.349495:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204992.349496:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204992.349497:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204992.349499:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497600:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204992.349501:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204992.349502:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204992.349503:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.349504:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204992.349507:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204992.349508:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204992.349510:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204992.349511:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204992.349513:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204992.349515:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543497600, portal 10 00000100:00000001:3.0:1788204992.349516:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204992.349517:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000100:00000040:3.0:1788204992.349521:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349526:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204992.349527:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204992.349529:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204992.349532:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543497600, offset 0 00000400:00000200:3.0:1788204992.349536:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204992.349540:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204992.349543:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204992.349545:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204992.349548:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204992.349551:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204992.349553:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204992.349556:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471236:471236:256:4294967295] 192.168.201.118@tcp LPNI seq info [471095:471095:8:4294967295] 00000400:00000200:3.0:1788204992.349560:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204992.349566:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204992.349571:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204992.349574:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204992.349577:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204992.349580:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204992.349584:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204992.349586:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788204992.349589:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204992.349591:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204992.349593:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204992.349595:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204992.349598:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204992.349599:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204992.349600:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204992.349603:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349608:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204992.349610:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349614:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349615:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204992.349617:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204992.349619:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204992.349621:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204992.349622:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321351188 00000100:00000001:3.0:1788204992.349624:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321352453 00000100:00000001:3.0:1788204992.349625:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321351188 00000100:00000001:3.0:1788204992.349627:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321352453 00000100:00000001:3.0:1788204992.349629:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204992.349630:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349631:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204992.349634:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204992/real 1788204992] req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204992.349638:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204992.349638:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204992.349639:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204992.349641:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204992.349641:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204992.349642:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204992.349644:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204992.349649:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349653:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204992.349654:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349655:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.349655:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204992.349657:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349661:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204992.349662:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349666:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204992.349667:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349668:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.349669:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349670:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204992.349672:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204992.349674:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204992.349676:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204992.349677:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204992.349680:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204992.349684:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497600:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204992.349686:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204992.349688:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349692:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204992.349692:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.349694:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204992.349695:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204992.349696:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204992.349697:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497728:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204992.349699:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204992.349700:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204992.349701:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.349702:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204992.349704:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204992.349705:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204992.349707:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204992.349709:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204992.349711:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204992.349713:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543497728, portal 4 00000100:00000001:3.0:1788204992.349714:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204992.349715:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:3.0:1788204992.349719:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349723:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204992.349725:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204992.349726:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204992.349727:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543497728, offset 0 00000400:00000200:3.0:1788204992.349730:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204992.349732:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204992.349734:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204992.349735:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204992.349737:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204992.349738:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204992.349739:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204992.349741:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471237:471237:256:4294967295] 192.168.201.118@tcp LPNI seq info [471096:471096:8:4294967295] 00000400:00000200:3.0:1788204992.349744:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204992.349748:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204992.349751:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204992.349752:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204992.349755:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204992.349757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204992.349759:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204992.349761:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788204992.349763:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204992.349764:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204992.349766:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204992.349768:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204992.349769:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204992.349770:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204992.349772:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204992.349774:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349778:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204992.349780:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349788:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349789:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204992.349790:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204992.349792:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204992.349794:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204992.349795:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321351197 00000100:00000001:3.0:1788204992.349796:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321352453 00000100:00000001:3.0:1788204992.349798:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321351197 00000100:00000001:3.0:1788204992.349799:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321352453 00000100:00000001:3.0:1788204992.349800:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204992.349801:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349802:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204992.349804:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204992/real 1788204992] req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204992.349808:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204992.349809:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204992.349810:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204992.349811:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204992.349812:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204992.349813:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204992.349815:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204992.349819:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349822:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204992.349823:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349824:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.349825:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204992.349826:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349831:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204992.349833:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349837:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204992.349838:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349838:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.349839:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349840:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204992.349841:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204992.349842:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204992.349844:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204992.349845:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204992.349847:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204992.349851:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497728:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204992.349853:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204992.349854:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349857:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204992.349858:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.349859:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204992.349860:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204992.349861:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204992.349863:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497856:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204992.349865:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204992.349866:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204992.349867:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204992.349868:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204992.349870:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204992.349871:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204992.349873:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204992.349875:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204992.349876:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204992.349878:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543497856, portal 4 00000100:00000001:3.0:1788204992.349879:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204992.349880:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:3.0:1788204992.349884:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349887:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204992.349889:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204992.349890:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204992.349891:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543497856, offset 0 00000400:00000200:3.0:1788204992.349893:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204992.349894:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204992.349896:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204992.349897:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204992.349899:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204992.349900:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204992.349902:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204992.349904:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471238:471238:256:4294967295] 192.168.201.118@tcp LPNI seq info [471097:471097:8:4294967295] 00000400:00000200:3.0:1788204992.349906:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204992.349910:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204992.349926:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204992.349928:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204992.349930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204992.349931:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:3.1:1788204992.349934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204992.349935:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:3.0:1788204992.349938:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204992.349940:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204992.349941:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204992.349944:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204992.349946:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204992.349947:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204992.349948:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204992.349950:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349955:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204992.349957:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205017 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349960:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349961:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204992.349963:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204992.349965:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204992.349966:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204992.349967:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321351220 00000100:00000001:3.0:1788204992.349969:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321352453 00000100:00000001:3.0:1788204992.349970:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321351220 00000100:00000001:3.0:1788204992.349972:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321352453 00000100:00000001:3.0:1788204992.349973:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204992.349974:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.349975:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204992.349977:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204992/real 1788204992] req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204992.349981:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204992.349982:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204992.349983:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204992.349984:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204992.349985:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204992.349987:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204992.349989:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204992.349993:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.349997:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204992.349997:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.349999:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.350000:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204992.350003:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.350009:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204992.350012:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.350018:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204992.350021:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.350023:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204992.350024:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.350025:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204992.350027:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204992.350030:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204992.350033:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204992.350036:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204992.350039:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204992.350047:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543497856:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204992.350051:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.350052:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204992.350055:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543497472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.350059:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204992.350060:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204992.350061:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204992.350063:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204992.350064:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204992.350065:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204992.350066:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204992.350068:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:3.0:1788204992.350070:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204992.350070:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204992.350071:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:3.0:1788204992.350073:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204992.350074:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.350075:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204992.350077:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543497600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.350081:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204992.350081:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204992.350082:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204992.350083:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204992.350084:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204992.350084:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204992.350085:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204992.350086:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:3.0:1788204992.350087:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204992.350088:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204992.350088:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:3.0:1788204992.350104:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204992.350105:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.350106:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204992.350108:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543497728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.350112:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204992.350113:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204992.350114:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204992.350116:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204992.350117:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204992.350118:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204992.350119:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204992.350120:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:3.0:1788204992.350121:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204992.350122:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204992.350122:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:3.0:1788204992.350123:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204992.350124:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.350125:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204992.350127:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77b80 x1875055543497856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205017 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204992.350131:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204992.350132:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204992.350133:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204992.350134:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204992.350134:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204992.350135:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204992.350136:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204992.350136:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788204992.350137:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204992.350138:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204992.350139:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77b80. 00000100:00000001:3.0:1788204992.350139:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204992.350140:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.350141:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204992.350143:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204992.350144:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204992.350145:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204992.350146:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204993.180206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204993.180207:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204993.180214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204993.180215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204993.180215:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204993.180217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204993.180220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204993.180224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204993.180227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204993.180229:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204993.180230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204993.180230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204993.180234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204993.180236:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204993.180237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204993.180237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204993.180240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204993.180243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204993.180243:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204993.180246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204993.180248:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204993.180250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204993.180250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204993.180253:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204993.180258:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204993.180261:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204993.180329:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204993.180342:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204993.180346:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788204993.692219:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321352453, at 4321352512 00000100:00000040:2.0:1788204993.692229:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321352453, at 4321352512 00000100:00000010:3.0:1788204993.692235:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcd80. 00000100:00000010:2.0:1788204993.692254:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fccc0. 00000100:00000001:3.0:1788204994.204169:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204994.204178:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204994.204180:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204994.204186:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204994.204189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204994.204191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204994.204192:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204994.204195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204994.204198:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204994.204205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788204994.204205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204994.204205:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204994.204212:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788204994.204215:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204994.204216:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204218:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204994.204228:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:1.0:1788204994.204234:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204994.204241:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204994.204242:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204994.204245:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204994.204248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204994.204249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204994.204254:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204994.204256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204994.204257:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204994.204260:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204994.204262:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788204994.204262:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204272:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204275:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204994.204277:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204278:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204994.204283:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788204994.204284:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788204994.204285:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204286:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788204994.204289:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204291:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204994.204293:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204994.204354:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204994.204363:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204994.204367:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204995.228221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204995.228221:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204995.228228:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204995.228229:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788204995.228233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204995.228233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204995.228234:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204995.228244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204995.228245:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204995.228245:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204995.228254:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204995.228257:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204995.228259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204995.228260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204995.228261:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204995.228267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204995.228268:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204995.228268:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204995.228274:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204995.228278:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204995.228279:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204995.228280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204995.228286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204995.228288:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204995.228294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204995.228296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204995.228350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204995.228353:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204995.228355:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788204996.252194:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204996.252201:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204996.252202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204996.252202:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204996.252208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204996.252208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788204996.252209:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204996.252213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204996.252218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204996.252219:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788204996.252219:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204996.252224:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204996.252228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204996.252229:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204996.252232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204996.252232:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204996.252232:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204996.252238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204996.252239:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204996.252240:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204996.252249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204996.252250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204996.252252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204996.252256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204996.252257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204996.252266:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204996.252308:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204996.252312:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204996.252314:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788204997.276212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204997.276212:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204997.276219:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204997.276219:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204997.276221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.276222:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204997.276226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204997.276228:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788204997.276231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204997.276232:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204997.276234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.276236:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204997.276237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204997.276241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.276241:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204997.276246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204997.276246:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.276246:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204997.276255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.276256:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204997.276267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204997.276271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204997.276277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204997.276279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204997.276284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204997.276287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204997.276354:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204997.276359:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204997.276362:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788204997.468210:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204997.468217:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788204997.468220:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204997.468222:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204997.468225:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204997.468231:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26683 00000400:00000200:1.0:1788204997.468238:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204997.468243:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204997.468246:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204997.468248:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468250:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204997.468252:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468253:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204997.468256:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204997.468257:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788204997.468260:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204997.468262:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788204997.468263:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788204997.468265:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788204997.468267:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468268:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204997.468269:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468270:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204997.468272:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788204997.468274:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468276:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204997.468279:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204997.468281:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204997.468283:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468284:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788204997.468286:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204997.468294:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000020:00000040:1.0:1788204997.468296:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788204997.468299:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204997.468300:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468302:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204997.468304:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204997.468308:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204997.468313:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8400. 00000100:00000001:1.0:1788204997.468318:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204997.468324:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204997.468330:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204997.468333:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204997.468336:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204997.468351:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788204997.468353:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204997.468355:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788204997.468356:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204997.468357:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204997.468358:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204997.468360:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26683 00000400:00000200:1.0:1788204997.468362:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788204997.468364:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204997.468365:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204997.468367:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468369:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204997.468371:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468373:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204997.468374:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204997.468376:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788204997.468376:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204997.468378:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788204997.468379:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788204997.468380:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788204997.468381:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788204997.468384:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788204997.468385:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204997.468389:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204997.468390:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788204997.468393:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468395:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204997.468397:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000040:3.0:1788204997.468400:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788204997.468401:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468404:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204997.468408:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204997.468413:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204997.468415:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788204997.468416:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468418:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788204997.468419:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788204997.468421:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204997.468425:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.468425:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788204997.468428:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.468430:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1788204997.468431:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788204997.468435:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788204997.468436:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204997.468438:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204997.468440:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468442:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788204997.468442:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543497984:192.168.201.118@tcp:250:kworker.0 02000000:00000001:1.0:1788204997.468445:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204997.468447:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.468449:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1788204997.468451:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb000. 02000000:00000001:3.0:1788204997.468453:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788204997.468454:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.468456:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.468459:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1788204997.468460:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204997.468466:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204997.468467:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788204997.468470:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 00000100:00000040:1.0:1788204997.468471:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788204997.468474:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468482:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.468483:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:1.0:1788204997.468486:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788204997.468487:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00080000:1.0:1788204997.468489:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788204997.468491:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204997.468492:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204997.468494:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:3.0:1788204997.468494:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00080000:1.0:1788204997.468498:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26683 00000100:00000200:3.0:1788204997.468502:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543497984, portal 25 00000400:00000200:1.0:1788204997.468503:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204997.468507:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1788204997.468508:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788204997.468510:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00080000:1.0:1788204997.468511:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204997.468515:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468517:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788204997.468519:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788204997.468521:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468525:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204997.468529:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:3.0:1788204997.468531:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1788204997.468532:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788204997.468535:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204997.468536:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000040:3.0:1788204997.468537:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000020:00000001:1.0:1788204997.468538:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000010:3.0:1788204997.468540:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000020:00000001:1.0:1788204997.468541:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000200:3.0:1788204997.468545:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543497984, offset 0 00000100:00000040:1.0:1788204997.468546:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468549:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204997.468550:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468554:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788204997.468554:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000020:00000040:1.0:1788204997.468557:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788204997.468560:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468561:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204997.468564:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788204997.468564:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:1.0:1788204997.468568:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788204997.468571:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.468572:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:1.0:1788204997.468573:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788204997.468576:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788204997.468579:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.468579:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788204997.468582:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.468584:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000010:1.0:1788204997.468586:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000400:00000200:3.0:1788204997.468588:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1788204997.468590:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000200:3.0:1788204997.468592:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 02000000:00000001:1.0:1788204997.468594:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204997.468596:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468598:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204997.468600:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788204997.468600:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471239:471239:256:4294967295] 192.168.201.118@tcp LPNI seq info [471098:471098:8:4294967295] 02000000:00000001:1.0:1788204997.468602:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204997.468606:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bbc00. 00000100:00000001:1.0:1788204997.468609:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.468610:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:1.0:1788204997.468615:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788204997.468620:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788204997.468623:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204997.468628:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788204997.468631:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:1.0:1788204997.468642:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788204997.468646:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:1.0:1788204997.468647:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788204997.468652:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788204997.468652:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204997.468653:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:3.0:1788204997.468654:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000100:00000001:1.0:1788204997.468656:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204997.468659:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26683 00000400:00000200:1.0:1788204997.468664:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.0:1788204997.468664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:1.0:1788204997.468670:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:3.0:1788204997.468670:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00080000:1.0:1788204997.468674:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204997.468677:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468680:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.1:1788204997.468681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:1.0:1788204997.468684:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788204997.468686:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00000001:1.0:1788204997.468687:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204997.468691:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204997.468693:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788204997.468695:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000100:3.0:1788204997.468696:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000040:1.0:1788204997.468697:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000800:00000010:3.0:1788204997.468699:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000020:00000001:1.0:1788204997.468700:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788204997.468703:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788204997.468705:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1788204997.468706:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468710:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204997.468713:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788204997.468714:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1788204997.468715:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788204997.468718:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:3.0:1788204997.468720:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000100:00080000:1.0:1788204997.468721:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:3.0:1788204997.468723:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:1.0:1788204997.468724:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788204997.468727:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788204997.468729:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204997.468729:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1788204997.468732:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468734:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:3.0:1788204997.468735:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788204997.468736:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788204997.468741:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468744:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204997.468749:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000020:00000040:1.0:1788204997.468752:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788204997.468752:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788204997.468755:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000040:3.0:1788204997.468757:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788204997.468758:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468761:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788204997.468762:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788204997.468763:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788204997.468765:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb800. 00000100:00000001:1.0:1788204997.468767:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.468774:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788204997.468775:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204997.468776:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204997.468783:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468784:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788204997.468788:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788204997.468789:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.468792:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321357573 00000100:00000040:1.0:1788204997.468796:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788204997.468798:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcd80. 00000100:00000001:3.0:1788204997.468805:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788204997.468809:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.468812:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321357573 00000100:00080000:1.0:1788204997.468813:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:3.0:1788204997.468816:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fccc0. 00000100:00080000:1.0:1788204997.468819:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788204997.468820:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788204997.468821:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788204997.468838:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788204997.468840:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788204997.468842:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26683 00000100:00000001:3.0:1788204997.468845:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788204997.468846:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204997.468848:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788204997.468850:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788204997.468852:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788204997.468854:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468857:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788204997.468858:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1788204997.468859:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468862:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788204997.468864:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788204997.468866:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00100000:3.0:1788204997.468867:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204997/real 1788204997] req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788204997.468868:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788204997.468870:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788204997.468873:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788204997.468876:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788204997.468882:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468885:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788204997.468888:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788204997.468890:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000001:3.0:1788204997.468890:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:1.0:1788204997.468894:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000001:3.0:1788204997.468894:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1788204997.468899:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:3.0:1788204997.468899:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00000001:1.0:1788204997.468903:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.468904:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 02000000:00000001:1.0:1788204997.468905:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:3.0:1788204997.468907:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 02000000:00000001:1.0:1788204997.468909:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204997.468910:0:256132:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1788204997.468913:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468916:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:3.0:1788204997.468917:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788204997.468918:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788204997.468923:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468926:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788204997.468932:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000100:00000200:3.0:1788204997.468933:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788204997.468934:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788204997.468938:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788204997.468940:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204997.468945:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788204997.468945:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1788204997.468946:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788204997.468948:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788204997.468949:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.468951:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:1.0:1788204997.468953:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9c00. 00000100:00000001:3.0:1788204997.468954:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204997.468958:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204997.468959:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788204997.468965:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788204997.468968:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204997.468974:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788204997.468975:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204997.468978:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204997.468985:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788204997.468990:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788204997.468993:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26693) 00000100:00000001:3.0:1788204997.469023:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204997.469025:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469026:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.469027:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469028:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204997.469030:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204997.469032:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204997.469035:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204997.469037:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204997.469039:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204997.469045:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543497984:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788204997.469047:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469049:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.469050:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543497984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469054:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204997.469056:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204997.469057:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204997.469059:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204997.469060:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204997.469061:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788204997.469062:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204997.469064:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:3.0:1788204997.469066:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204997.469067:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204997.469069:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788204997.469074:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204997.469075:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469076:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469082:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204997.469083:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469084:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204997.469085:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204997.469086:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204997.469088:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469120:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204997.469121:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.469122:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204997.469123:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204997.469125:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204997.469126:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498112:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204997.469128:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204997.469130:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204997.469131:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.469132:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204997.469134:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204997.469135:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.469138:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204997.469140:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204997.469142:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204997.469144:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543498112, portal 10 00000100:00000001:3.0:1788204997.469145:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204997.469146:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788204997.469150:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469154:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204997.469155:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204997.469157:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204997.469158:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543498112, offset 0 00000400:00000200:3.0:1788204997.469160:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204997.469163:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204997.469166:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204997.469168:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204997.469169:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204997.469171:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204997.469172:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204997.469174:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471240:471240:256:4294967295] 192.168.201.118@tcp LPNI seq info [471099:471099:8:4294967295] 00000400:00000200:3.0:1788204997.469177:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204997.469181:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204997.469184:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204997.469185:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204997.469194:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204997.469197:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204997.469202:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204997.469204:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1788204997.469207:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204997.469208:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204997.469211:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204997.469214:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204997.469217:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204997.469219:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204997.469221:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204997.469225:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469233:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.469236:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469241:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469242:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204997.469244:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204997.469247:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788204997.469249:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204997.469251:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321356307 00000100:00000001:3.0:1788204997.469253:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321357573 00000100:00000001:3.0:1788204997.469255:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321356307 00000100:00000001:3.0:1788204997.469256:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321357573 00000100:00000001:3.0:1788204997.469257:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204997.469258:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469259:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204997.469261:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204997/real 1788204997] req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204997.469266:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204997.469266:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204997.469267:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204997.469269:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204997.469270:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204997.469271:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204997.469273:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204997.469291:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469296:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204997.469298:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469300:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.469301:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204997.469305:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469312:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204997.469316:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469323:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204997.469325:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469327:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.469328:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469330:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204997.469332:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204997.469335:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204997.469338:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204997.469339:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204997.469342:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204997.469347:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498112:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204997.469349:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204997.469351:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469355:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204997.469356:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.469357:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204997.469358:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204997.469359:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204997.469361:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498240:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204997.469362:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204997.469364:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204997.469364:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.469365:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204997.469368:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204997.469369:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.469371:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204997.469373:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204997.469374:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204997.469376:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543498240, portal 10 00000100:00000001:3.0:1788204997.469378:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204997.469379:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788204997.469382:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469388:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204997.469391:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204997.469392:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204997.469395:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543498240, offset 0 00000400:00000200:3.0:1788204997.469398:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204997.469402:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204997.469405:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204997.469408:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204997.469410:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204997.469413:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204997.469415:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204997.469418:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471241:471241:256:4294967295] 192.168.201.118@tcp LPNI seq info [471100:471100:8:4294967295] 00000400:00000200:3.0:1788204997.469422:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204997.469428:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204997.469431:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204997.469434:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204997.469436:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204997.469438:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204997.469440:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204997.469442:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1788204997.469444:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204997.469445:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204997.469447:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204997.469449:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204997.469451:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204997.469452:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204997.469454:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204997.469456:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469464:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.469466:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469472:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469473:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204997.469476:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204997.469479:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788204997.469482:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204997.469483:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321356308 00000100:00000001:3.0:1788204997.469486:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321357573 00000100:00000001:3.0:1788204997.469488:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321356308 00000100:00000001:3.0:1788204997.469490:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321357573 00000100:00000001:3.0:1788204997.469492:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204997.469494:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469496:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204997.469499:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204997/real 1788204997] req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204997.469507:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204997.469509:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204997.469510:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204997.469512:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204997.469513:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204997.469514:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204997.469517:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204997.469522:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469528:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204997.469530:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469532:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.469533:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204997.469536:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469543:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204997.469547:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469553:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204997.469555:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469557:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.469558:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469560:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204997.469562:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204997.469564:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204997.469567:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204997.469569:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204997.469572:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204997.469580:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498240:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788204997.469584:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204997.469586:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469592:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204997.469593:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.469596:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204997.469597:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204997.469599:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204997.469602:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498368:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204997.469604:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204997.469606:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204997.469607:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.469609:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204997.469612:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204997.469614:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.469617:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204997.469620:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204997.469623:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204997.469626:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543498368, portal 4 00000100:00000001:3.0:1788204997.469627:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204997.469629:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788204997.469634:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469640:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204997.469642:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204997.469644:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204997.469647:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543498368, offset 0 00000400:00000200:3.0:1788204997.469649:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204997.469652:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204997.469654:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204997.469656:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204997.469658:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204997.469659:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204997.469661:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204997.469663:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471242:471242:256:4294967295] 192.168.201.118@tcp LPNI seq info [471101:471101:8:4294967295] 00000400:00000200:3.0:1788204997.469665:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204997.469669:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204997.469672:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204997.469674:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204997.469677:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204997.469678:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204997.469681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204997.469682:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1788204997.469684:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204997.469686:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204997.469687:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204997.469689:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204997.469691:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204997.469692:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204997.469693:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204997.469695:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469700:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.469702:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469705:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469706:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204997.469708:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204997.469710:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788204997.469712:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204997.469713:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321356317 00000100:00000001:3.0:1788204997.469715:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321357573 00000100:00000001:3.0:1788204997.469716:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321356317 00000100:00000001:3.0:1788204997.469717:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321357573 00000100:00000001:3.0:1788204997.469718:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204997.469719:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469721:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204997.469723:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204997/real 1788204997] req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204997.469727:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204997.469728:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204997.469729:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204997.469731:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204997.469732:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204997.469734:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204997.469737:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204997.469744:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469750:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204997.469751:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469753:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.469754:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204997.469757:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469764:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204997.469768:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469774:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204997.469777:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469778:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.469780:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469781:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204997.469783:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204997.469785:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204997.469789:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204997.469791:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204997.469795:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204997.469808:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498368:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204997.469811:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788204997.469814:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469819:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788204997.469820:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.469822:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788204997.469835:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788204997.469836:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788204997.469839:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498496:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204997.469841:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788204997.469842:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788204997.469843:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788204997.469844:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788204997.469846:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204997.469848:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788204997.469851:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788204997.469853:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788204997.469856:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788204997.469859:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543498496, portal 4 00000100:00000001:3.0:1788204997.469861:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788204997.469862:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788204997.469867:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469873:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788204997.469875:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788204997.469878:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788204997.469880:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543498496, offset 0 00000400:00000200:3.0:1788204997.469883:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788204997.469887:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204997.469890:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788204997.469893:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788204997.469896:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788204997.469898:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788204997.469900:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788204997.469903:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471243:471243:256:4294967295] 192.168.201.118@tcp LPNI seq info [471102:471102:8:4294967295] 00000400:00000200:3.0:1788204997.469907:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788204997.469912:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788204997.469916:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204997.469918:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788204997.469921:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788204997.469922:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:3.1:1788204997.469925:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788204997.469926:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:3.0:1788204997.469928:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788204997.469930:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788204997.469931:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788204997.469933:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788204997.469935:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788204997.469936:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788204997.469938:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788204997.469940:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469945:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.469946:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205022 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469950:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469951:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788204997.469953:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788204997.469955:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788204997.469957:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788204997.469958:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321356340 00000100:00000001:3.0:1788204997.469960:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321357573 00000100:00000001:3.0:1788204997.469961:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321356340 00000100:00000001:3.0:1788204997.469962:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321357573 00000100:00000001:3.0:1788204997.469963:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788204997.469964:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.469965:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788204997.469967:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788204997/real 1788204997] req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788204997.469971:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788204997.469972:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788204997.469973:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788204997.469975:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788204997.469976:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788204997.469977:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788204997.469978:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788204997.469985:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.469991:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788204997.469992:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.469994:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.469995:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788204997.469999:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.470005:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788204997.470008:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.470014:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788204997.470017:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.470018:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788204997.470020:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.470021:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788204997.470023:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788204997.470025:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788204997.470029:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788204997.470031:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788204997.470035:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788204997.470041:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498496:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788204997.470044:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.470045:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.470047:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543498112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.470051:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204997.470052:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204997.470053:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788204997.470055:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204997.470056:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204997.470057:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204997.470058:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204997.470059:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:3.0:1788204997.470061:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204997.470063:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204997.470064:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788204997.470067:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204997.470068:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.470069:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.470072:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543498240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.470083:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204997.470085:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204997.470086:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788204997.470089:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204997.470111:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204997.470112:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788204997.470114:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204997.470115:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:3.0:1788204997.470116:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204997.470117:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204997.470118:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788204997.470119:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204997.470120:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.470121:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.470124:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543498368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.470128:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204997.470129:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204997.470130:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788204997.470131:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204997.470132:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204997.470133:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204997.470134:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204997.470136:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788204997.470137:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204997.470138:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204997.470138:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788204997.470142:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204997.470143:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.470144:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788204997.470145:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543498496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205022 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788204997.470149:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788204997.470150:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788204997.470152:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788204997.470153:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788204997.470153:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788204997.470154:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788204997.470155:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788204997.470156:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:3.0:1788204997.470157:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788204997.470158:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788204997.470159:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788204997.470161:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788204997.470162:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.470163:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204997.470165:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204997.470166:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204997.470167:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788204997.470168:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204998.300170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788204998.300170:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204998.300173:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204998.300179:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204998.300179:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204998.300180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204998.300183:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204998.300186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788204998.300189:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204998.300190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204998.300190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204998.300195:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788204998.300197:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788204998.300198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204998.300200:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204998.300202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204998.300202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788204998.300206:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204998.300207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204998.300209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788204998.300210:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204998.300212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204998.300213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204998.300216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788204998.300216:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204998.300222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204998.300265:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204998.300268:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204998.300270:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788204998.812181:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321357573, at 4321357632 00000100:00000040:3.0:1788204998.812181:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321357573, at 4321357632 00000100:00000010:2.0:1788204998.812194:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcd80. 00000100:00000010:3.0:1788204998.812197:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fccc0. 00000100:00000001:3.0:1788204999.324182:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788204999.324184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788204999.324188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788204999.324191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788204999.324191:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788204999.324195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204999.324196:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788204999.324196:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204999.324200:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1788204999.324203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788204999.324203:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788204999.324205:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204999.324208:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788204999.324209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788204999.324211:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788204999.324212:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204999.324213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204999.324214:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204999.324216:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788204999.324218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788204999.324220:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204999.324220:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788204999.324221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788204999.324227:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788204999.324227:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788204999.324228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204999.324236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204999.324239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204999.324243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204999.324245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788204999.324250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788204999.324251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788204999.324287:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788204999.324290:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788204999.324292:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205000.348156:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205000.348162:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205000.348163:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205000.348168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205000.348168:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205000.348173:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.348173:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205000.348177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205000.348177:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205000.348178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205000.348180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.348180:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205000.348184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205000.348184:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205000.348185:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205000.348187:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205000.348189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205000.348190:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205000.348191:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205000.348193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205000.348196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205000.348196:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205000.348200:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205000.348202:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205000.348203:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205000.348205:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205000.348254:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205000.348259:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205000.348263:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205001.372187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205001.372187:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205001.372191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205001.372193:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205001.372194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205001.372194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205001.372196:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205001.372197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205001.372201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205001.372202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205001.372203:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205001.372204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205001.372209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205001.372210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205001.372210:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205001.372214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205001.372214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205001.372215:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205001.372218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205001.372219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205001.372220:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205001.372221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205001.372224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205001.372225:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205001.372230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205001.372230:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205001.372283:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205001.372288:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205001.372292:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205002.396182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205002.396182:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205002.396189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205002.396189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205002.396189:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205002.396191:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.396192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205002.396194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205002.396198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205002.396199:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205002.396201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205002.396204:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205002.396204:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205002.396205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.396209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205002.396209:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205002.396211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205002.396215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205002.396215:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205002.396216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.396218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205002.396219:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205002.396222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205002.396222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205002.396222:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205002.396227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205002.396278:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205002.396282:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205002.396286:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205002.588212:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205002.588223:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205002.588225:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205002.588228:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205002.588232:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205002.588239:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26688 00000400:00000200:3.0:1788205002.588247:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205002.588254:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205002.588257:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205002.588260:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588263:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205002.588265:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588268:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205002.588272:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205002.588274:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205002.588276:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205002.588280:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205002.588283:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205002.588285:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205002.588288:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588290:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205002.588291:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588293:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205002.588296:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205002.588299:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588301:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205002.588303:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205002.588307:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205002.588310:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205002.588312:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205002.588315:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205002.588323:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205002.588327:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205002.588330:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205002.588332:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205002.588334:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205002.588336:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205002.588340:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205002.588347:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:3.0:1788205002.588353:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205002.588360:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205002.588373:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205002.588377:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205002.588382:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205002.588403:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205002.588407:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205002.588410:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205002.588411:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205002.588413:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205002.588417:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205002.588420:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26688 00000100:00000001:1.0:1788205002.588423:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205002.588427:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205002.588430:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205002.588436:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205002.588437:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205002.588445:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205002.588447:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1788205002.588449:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205002.588450:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205002.588456:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1788205002.588457:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205002.588459:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205002.588461:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788205002.588462:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205002.588465:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00100000:1.0:1788205002.588467:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543498624:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205002.588470:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205002.588473:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1788205002.588474:0:409030:0:(genops.c:814:class_conn2export()) Process entered 02000000:00000001:1.0:1788205002.588478:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1788205002.588478:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 02000000:00000001:1.0:1788205002.588481:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205002.588482:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1788205002.588484:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1788205002.588486:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205002.588489:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 02000000:00000010:1.0:1788205002.588490:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba400. 00000020:00000001:3.0:1788205002.588494:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 02000000:00000001:1.0:1788205002.588495:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205002.588502:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205002.588503:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205002.588506:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1788205002.588508:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000100:00000040:3.0:1788205002.588511:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588514:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000010:1.0:1788205002.588516:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000020:00000040:3.0:1788205002.588519:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000200:1.0:1788205002.588523:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543498624, portal 25 00000100:00080000:3.0:1788205002.588524:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205002.588526:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205002.588529:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000001:3.0:1788205002.588530:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205002.588534:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.588540:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788205002.588541:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205002.588542:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205002.588545:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205002.588548:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543498624, offset 0 02000000:00000001:3.0:1788205002.588551:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205002.588556:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205002.588556:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205002.588560:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205002.588564:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205002.588567:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205002.588569:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205002.588573:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000400:00000200:1.0:1788205002.588576:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:3.0:1788205002.588577:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205002.588582:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205002.588584:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.588587:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788205002.588587:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205002.588591:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1788205002.588593:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1788205002.588594:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.588598:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000010:3.0:1788205002.588599:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413800. 00000400:00000200:1.0:1788205002.588602:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788205002.588605:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.588606:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205002.588611:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471244:471244:256:4294967295] 192.168.201.118@tcp LPNI seq info [471103:471103:8:4294967295] 00000100:00100000:3.0:1788205002.588615:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205002.588618:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205002.588628:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205002.588629:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788205002.588633:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205002.588636:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205002.588641:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000100:00000040:3.0:1788205002.588641:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1788205002.588651:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205002.588656:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00000001:3.0:1788205002.588658:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788205002.588662:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:3.0:1788205002.588663:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1788205002.588667:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000100:00080000:3.0:1788205002.588669:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1788205002.588672:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588673:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:1.0:1788205002.588675:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000100:00080000:3.0:1788205002.588676:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205002.588680:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205002.588681:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1788205002.588687:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26688 00000400:00000200:1.0:1788205002.588688:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205002.588694:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000200:3.0:1788205002.588696:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205002.588698:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000400:00000200:3.0:1788205002.588702:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205002.588704:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1788205002.588706:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205002.588713:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000200:1.0:1788205002.588714:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205002.588715:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205002.588720:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588725:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205002.588729:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205002.588731:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000100:00000001:1.0:1788205002.588748:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205002.588754:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205002.588759:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205002.588764:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000001:1.0:1788205002.588766:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.588769:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1788205002.588769:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205002.588773:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788205002.588777:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205002.588778:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205002.588782:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205002.588784:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205002.588786:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321362693 00000100:00000040:3.0:1788205002.588789:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000010:1.0:1788205002.588792:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647960. 00000100:00000001:3.0:1788205002.588793:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205002.588797:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000001:1.0:1788205002.588798:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00080000:3.0:1788205002.588801:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205002.588805:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205002.588807:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205002.588810:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321362693 02000000:00000001:3.0:1788205002.588810:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205002.588813:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205002.588815:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16472a0. 00000100:00000001:3.0:1788205002.588817:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205002.588818:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205002.588822:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1788205002.588822:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788205002.588826:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788205002.588828:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.588829:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788205002.588833:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205002.588835:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205002/real 1788205002] req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788205002.588839:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205002.588845:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000001:1.0:1788205002.588846:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205002.588849:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788205002.588850:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1788205002.588851:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205002.588855:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 02000000:00000001:3.0:1788205002.588855:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205002.588859:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205002.588860:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205002.588862:0:256132:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1788205002.588863:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1788205002.588866:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205002.588867:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205002.588874:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412400. 00000100:00000200:1.0:1788205002.588875:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205002.588880:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.588881:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205002.588884:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.588886:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.588887:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205002.588889:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205002.588892:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205002.588912:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205002.588913:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205002.588916:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205002.588921:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205002.588924:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.588941:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205002.588944:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.588946:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205002.588949:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205002.588950:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.588953:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205002.588955:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205002.588957:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205002.588958:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205002.588961:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205002.588963:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205002.588966:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205002.588966:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26688 00000100:00080000:1.0:1788205002.588972:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000400:00000200:3.0:1788205002.588975:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:1.0:1788205002.588979:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000400:00000200:3.0:1788205002.588982:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205002.588983:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1788205002.588988:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205002.588991:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205002.588995:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00100000:1.0:1788205002.588999:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543498624:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205002.589002:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589005:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205002.589008:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205002.589009:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543498624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205002.589014:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205002.589017:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788205002.589019:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:1.0:1788205002.589020:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205002.589024:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba400. 00000020:00000001:3.0:1788205002.589024:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205002.589027:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 02000000:00000001:1.0:1788205002.589028:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205002.589030:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000001:1.0:1788205002.589031:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205002.589033:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000040:1.0:1788205002.589034:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205002.589037:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000020:00000001:3.0:1788205002.589039:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 02000000:00000010:1.0:1788205002.589040:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205002.589045:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1788205002.589046:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 02000000:00000001:1.0:1788205002.589047:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205002.589050:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000040:3.0:1788205002.589052:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205002.589055:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205002.589057:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205002.589060:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205002.589062:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589063:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205002.589065:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205002.589069:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1788205002.589070:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000001:1.0:1788205002.589072:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205002.589075:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205002.589076:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205002.589078:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205002.589080:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589081:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788205002.589085:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205002.589087:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205002.589089:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205002.589116:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205002.589117:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205002.589123:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205002.589124:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205002.589127:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205002.589128:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1788205002.589131:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 02000000:00000001:1.0:1788205002.589132:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788205002.589137:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205002.589139:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498752:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205002.589141:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589142:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205002.589146:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205002.589147:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205002.589147:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 02000000:00000001:1.0:1788205002.589151:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1788205002.589153:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000010:1.0:1788205002.589155:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba400. 02000000:00000001:1.0:1788205002.589158:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205002.589158:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205002.589161:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.589164:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205002.589166:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1788205002.589167:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 02000000:00000001:3.0:1788205002.589168:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1788205002.589171:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 02000000:00000001:3.0:1788205002.589172:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205002.589174:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543498752, portal 10 00000100:00000001:1.0:1788205002.589177:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205002.589178:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 02000000:00000010:3.0:1788205002.589178:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788205002.589185:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205002.589187:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205002.589194:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205002.589195:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205002.589200:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205002.589204:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205002.589207:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543498752, offset 0 00000100:00000001:3.0:1788205002.589208:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205002.589211:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205002.589212:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205002.589218:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205002.589225:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205002.589233:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.589234:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00080000:3.0:1788205002.589238:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205002.589240:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1788205002.589244:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:1.0:1788205002.589247:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:3.0:1788205002.589248:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205002.589252:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788205002.589253:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788205002.589256:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205002.589258:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00080000:3.0:1788205002.589262:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26688 00000400:00000200:1.0:1788205002.589266:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471245:471245:256:4294967295] 192.168.201.118@tcp LPNI seq info [471104:471104:8:4294967295] 00000400:00000200:3.0:1788205002.589272:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205002.589273:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205002.589279:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:1.0:1788205002.589284:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:3.0:1788205002.589285:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1788205002.589292:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000040:3.0:1788205002.589293:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000010:1.0:1788205002.589296:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000100:00000001:3.0:1788205002.589297:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205002.589303:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.0:1788205002.589304:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205002.589307:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.0:1788205002.589308:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000020:00000001:3.0:1788205002.589313:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:1.1:1788205002.589316:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:3.0:1788205002.589317:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000800:00000200:1.1:1788205002.589319:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000020:00000001:3.0:1788205002.589320:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000100:1.0:1788205002.589324:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000040:3.0:1788205002.589324:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000800:00000010:1.0:1788205002.589327:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000020:00000001:3.0:1788205002.589329:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000200:1.0:1788205002.589330:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205002.589335:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1788205002.589335:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000200:1.0:1788205002.589339:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205002.589342:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000040:3.0:1788205002.589343:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205002.589346:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1788205002.589347:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1788205002.589351:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205002.589353:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205002.589359:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205002.589362:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1788205002.589363:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000040:1.0:1788205002.589367:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205002.589369:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205002.589375:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205002.589380:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205002.589381:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589384:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1788205002.589385:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205002.589387:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205002.589392:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 02000000:00000001:3.0:1788205002.589392:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205002.589397:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205002.589398:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205002.589401:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321361427 00000008:00100000:3.0:1788205002.589402:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205002.589403:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321362693 00000100:00000001:1.0:1788205002.589407:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321361427 00000008:00000001:3.0:1788205002.589408:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589409:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321362693 00000100:00000001:1.0:1788205002.589413:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205002.589413:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589417:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589419:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000010:3.0:1788205002.589421:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205002.589424:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205002.589426:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205002.589428:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205002.589430:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205002/real 1788205002] req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205002.589431:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205002.589434:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205002.589435:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205002.589441:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000400:00000001:1.0:1788205002.589443:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205002.589446:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205002.589448:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000100:00000001:3.0:1788205002.589448:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.589450:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205002.589453:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205002.589454:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1788205002.589457:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205002.589459:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205002.589481:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205002.589483:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205002.589488:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205002.589488:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589490:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589493:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.589494:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205002.589497:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205002.589498:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589520:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205002.589526:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205002.589527:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589534:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1788205002.589534:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26699) 00000100:00000001:1.0:1788205002.589538:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589540:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.589541:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589542:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205002.589544:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205002.589545:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205002.589547:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205002.589549:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205002.589554:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205002.589561:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498752:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205002.589565:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205002.589568:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589574:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205002.589575:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205002.589577:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205002.589579:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205002.589581:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205002.589583:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498880:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205002.589585:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205002.589586:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205002.589587:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205002.589588:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205002.589591:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205002.589592:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.589594:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205002.589596:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205002.589598:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205002.589600:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543498880, portal 10 00000100:00000001:1.0:1788205002.589601:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205002.589602:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205002.589605:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589609:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205002.589611:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205002.589612:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205002.589613:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543498880, offset 0 00000400:00000200:1.0:1788205002.589615:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205002.589618:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205002.589620:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205002.589621:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205002.589623:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205002.589625:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205002.589626:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205002.589628:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471246:471246:256:4294967295] 192.168.201.118@tcp LPNI seq info [471105:471105:8:4294967295] 00000400:00000200:1.0:1788205002.589631:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205002.589635:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205002.589637:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205002.589639:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205002.589641:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205002.589643:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788205002.589645:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205002.589647:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1788205002.589649:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205002.589650:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205002.589652:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205002.589654:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205002.589656:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205002.589657:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205002.589659:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205002.589661:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589666:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205002.589667:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589671:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589672:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205002.589674:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205002.589675:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205002.589677:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205002.589678:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321361428 00000100:00000001:1.0:1788205002.589680:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321362693 00000100:00000001:1.0:1788205002.589681:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321361428 00000100:00000001:1.0:1788205002.589682:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321362693 00000100:00000001:1.0:1788205002.589683:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205002.589685:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589686:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205002.589688:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205002/real 1788205002] req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205002.589691:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205002.589692:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205002.589693:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205002.589694:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205002.589695:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205002.589697:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205002.589700:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205002.589707:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589713:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205002.589715:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589716:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.589717:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205002.589721:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589727:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205002.589731:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589750:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205002.589752:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589753:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.589754:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589755:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205002.589756:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205002.589758:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205002.589760:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205002.589762:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205002.589764:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205002.589769:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543498880:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205002.589770:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589772:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205002.589774:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543498752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589777:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205002.589779:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205002.589780:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:1.0:1788205002.589781:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205002.589782:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205002.589783:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205002.589785:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205002.589786:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:1.0:1788205002.589787:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205002.589788:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205002.589789:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205002.589791:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205002.589791:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589792:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205002.589794:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543498880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589797:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205002.589798:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205002.589799:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205002.589800:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205002.589801:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205002.589802:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205002.589803:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205002.589803:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:1.0:1788205002.589805:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205002.589805:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205002.589806:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205002.589807:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205002.589808:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589809:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589811:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205002.589812:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589813:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205002.589813:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205002.589814:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205002.589816:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589819:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205002.589821:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205002.589822:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205002.589823:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205002.589824:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205002.589826:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499008:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205002.589828:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205002.589829:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205002.589830:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205002.589831:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205002.589833:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205002.589834:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.589836:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205002.589837:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205002.589840:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205002.589841:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543499008, portal 4 00000100:00000001:1.0:1788205002.589842:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205002.589843:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205002.589847:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589850:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205002.589852:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205002.589853:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205002.589854:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543499008, offset 0 00000400:00000200:1.0:1788205002.589856:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205002.589858:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205002.589861:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205002.589862:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205002.589864:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205002.589866:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205002.589867:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205002.589869:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471247:471247:256:4294967295] 192.168.201.118@tcp LPNI seq info [471106:471106:8:4294967295] 00000400:00000200:1.0:1788205002.589871:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205002.589875:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205002.589878:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205002.589880:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205002.589882:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205002.589884:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788205002.589887:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205002.589888:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1788205002.589890:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205002.589892:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205002.589893:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205002.589895:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205002.589897:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205002.589898:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205002.589899:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205002.589901:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589906:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205002.589907:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589911:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589912:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205002.589915:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205002.589917:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205002.589921:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205002.589922:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321361437 00000100:00000001:1.0:1788205002.589924:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321362693 00000100:00000001:1.0:1788205002.589926:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321361437 00000100:00000001:1.0:1788205002.589928:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321362693 00000100:00000001:1.0:1788205002.589930:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205002.589931:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.589933:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205002.589937:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205002/real 1788205002] req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205002.589945:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205002.589946:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205002.589948:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205002.589950:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205002.589952:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205002.589954:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205002.589957:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205002.589963:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589967:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205002.589968:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589969:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.589969:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205002.589972:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589976:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205002.589978:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.589981:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205002.589983:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589983:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.589984:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.589985:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205002.589986:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205002.589988:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205002.589989:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205002.589990:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205002.589993:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205002.589997:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499008:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205002.589999:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205002.590001:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590004:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205002.590005:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205002.590006:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205002.590007:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205002.590009:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205002.590010:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499136:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205002.590012:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205002.590013:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205002.590014:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205002.590015:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205002.590017:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba400. 02000000:00000001:1.0:1788205002.590018:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205002.590020:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205002.590022:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205002.590023:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205002.590025:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543499136, portal 4 00000100:00000001:1.0:1788205002.590026:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205002.590027:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205002.590030:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590034:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205002.590036:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205002.590037:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205002.590038:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543499136, offset 0 00000400:00000200:1.0:1788205002.590041:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205002.590043:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205002.590045:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205002.590047:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205002.590048:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205002.590050:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205002.590051:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205002.590053:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471248:471248:256:4294967295] 192.168.201.118@tcp LPNI seq info [471107:471107:8:4294967295] 00000400:00000200:1.0:1788205002.590056:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205002.590059:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205002.590062:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205002.590063:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205002.590066:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205002.590067:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1788205002.590069:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205002.590071:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1788205002.590073:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205002.590074:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205002.590076:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205002.590078:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205002.590080:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205002.590082:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205002.590083:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205002.590086:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590110:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205002.590114:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205027 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590120:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.590122:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205002.590125:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205002.590128:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205002.590131:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205002.590132:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321361461 00000100:00000001:1.0:1788205002.590134:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321362693 00000100:00000001:1.0:1788205002.590135:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321361461 00000100:00000001:1.0:1788205002.590136:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321362693 00000100:00000001:1.0:1788205002.590138:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205002.590139:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.590140:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205002.590142:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205002/real 1788205002] req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205002.590147:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205002.590147:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205002.590149:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205002.590150:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205002.590151:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205002.590152:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205002.590154:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205002.590158:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590162:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205002.590163:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.590164:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.590164:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205002.590166:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590170:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205002.590171:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590175:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205002.590177:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.590177:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205002.590178:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.590179:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205002.590180:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205002.590182:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205002.590183:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205002.590185:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205002.590187:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205002.590190:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499136:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205002.590193:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.590194:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205002.590195:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543499008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590199:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205002.590200:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205002.590201:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205002.590202:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205002.590203:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205002.590204:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205002.590205:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205002.590207:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205002.590209:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205002.590210:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205002.590211:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205002.590213:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205002.590215:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.590216:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205002.590219:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543499136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205027 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205002.590230:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205002.590231:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205002.590233:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:1.0:1788205002.590235:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205002.590236:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205002.590237:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205002.590239:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205002.590240:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205002.590242:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205002.590243:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205002.590244:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205002.590246:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205002.590247:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.590249:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205002.590252:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205002.590253:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205002.590254:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205002.590256:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205003.420182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205003.420182:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205003.420189:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205003.420190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205003.420191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205003.420192:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205003.420193:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205003.420196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205003.420199:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205003.420200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205003.420201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205003.420205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205003.420206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205003.420206:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205003.420210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205003.420211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205003.420212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205003.420212:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205003.420218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205003.420218:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205003.420219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205003.420222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205003.420222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205003.420223:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205003.420227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205003.420228:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205003.420277:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205003.420279:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205003.420282:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205003.932187:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321362693, at 4321362752 00000100:00000040:3.0:1788205003.932187:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321362693, at 4321362752 00000100:00000010:3.0:1788205003.932201:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00000100:00000010:2.0:1788205003.932204:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647960. 00000100:00000001:1.0:1788205004.444184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205004.444184:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205004.444192:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205004.444193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205004.444194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205004.444196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205004.444197:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205004.444203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205004.444204:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205004.444207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205004.444208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205004.444209:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205004.444213:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205004.444213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205004.444214:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205004.444215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205004.444218:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205004.444219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205004.444220:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205004.444225:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205004.444225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205004.444225:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205004.444232:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444233:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205004.444237:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444240:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444243:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205004.444250:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205004.444252:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205004.444253:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444255:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205004.444257:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444258:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444260:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205004.444264:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205004.444266:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205004.444267:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444269:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205004.444270:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444271:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205004.444273:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205004.444335:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205004.444339:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205004.444342:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205005.468242:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205005.468242:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205005.468244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205005.468252:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205005.468253:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205005.468256:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205005.468256:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205005.468260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205005.468263:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205005.468263:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205005.468272:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205005.468273:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205005.468277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205005.468279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205005.468282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205005.468284:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205005.468285:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205005.468290:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205005.468292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205005.468292:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205005.468296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205005.468300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205005.468301:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205005.468309:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205005.468311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205005.468315:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205005.468380:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205005.468384:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205005.468387:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205006.492202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205006.492202:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205006.492208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205006.492212:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205006.492213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205006.492213:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205006.492216:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205006.492217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205006.492220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205006.492222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205006.492224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205006.492224:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205006.492226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205006.492228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205006.492232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205006.492232:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205006.492234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205006.492237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205006.492237:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.492237:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205006.492241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205006.492242:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205006.492244:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205006.492246:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205006.492250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205006.492250:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205006.492313:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205006.492317:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205006.492319:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205007.516201:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205007.516201:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205007.516211:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205007.516212:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205007.516214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.516215:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205007.516221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205007.516222:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205007.516226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205007.516227:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205007.516231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205007.516232:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.516234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205007.516237:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205007.516238:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205007.516241:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.516242:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.516245:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205007.516344:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205007.516348:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205007.516352:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205007.516353:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205007.516355:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205007.516356:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205007.516358:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205007.516359:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205007.516421:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205007.516425:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205007.516428:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205007.708221:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205007.708231:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205007.708234:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205007.708237:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205007.708268:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205007.708275:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26694 00000400:00000200:3.0:1788205007.708286:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205007.708293:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205007.708296:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205007.708299:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708301:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.708303:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708304:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205007.708308:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205007.708309:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205007.708312:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205007.708315:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205007.708317:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205007.708318:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205007.708320:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708321:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.708322:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708323:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205007.708325:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205007.708327:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708329:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205007.708331:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205007.708334:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205007.708336:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.708338:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205007.708340:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205007.708350:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205007.708352:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205007.708356:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205007.708357:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.708360:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205007.708362:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205007.708367:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205007.708374:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788205007.708381:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205007.708390:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205007.708397:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205007.708401:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205007.708404:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205007.708425:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205007.708428:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205007.708430:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205007.708432:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205007.708433:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205007.708435:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205007.708437:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26694 00000400:00000200:3.0:1788205007.708440:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205007.708441:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205007.708443:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205007.708445:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708446:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.708447:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708448:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205007.708450:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205007.708450:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205007.708451:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205007.708453:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205007.708454:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205007.708455:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205007.708458:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708460:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.708462:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708463:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205007.708465:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205007.708466:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708468:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205007.708469:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205007.708475:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205007.708476:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.708477:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205007.708479:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788205007.708481:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205007.708485:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.708486:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205007.708486:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.708491:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:3.0:1788205007.708491:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788205007.708494:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205007.708496:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205007.708497:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.708498:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205007.708499:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205007.708500:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205007.708505:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000040:1.0:1788205007.708509:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205007.708510:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205007.708522:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205007.708523:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205007.708526:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.708532:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205007.708536:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788205007.708537:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205007.708541:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.708542:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205007.708548:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543499264:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205007.708552:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205007.708557:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1788205007.708557:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205007.708561:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.708567:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788205007.708572:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205007.708575:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba400. 00000100:00080000:3.0:1788205007.708575:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788205007.708580:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205007.708581:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205007.708584:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205007.708586:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205007.708588:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205007.708591:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26694 00000400:00000200:1.0:1788205007.708593:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205007.708597:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205007.708599:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:3.0:1788205007.708600:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205007.708604:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205007.708607:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708608:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.708610:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:1.0:1788205007.708611:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205007.708612:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205007.708617:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205007.708618:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205007.708619:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205007.708620:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205007.708621:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000200:1.0:1788205007.708623:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543499264, portal 25 00000020:00000001:3.0:1788205007.708623:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788205007.708629:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1788205007.708629:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205007.708634:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000001:3.0:1788205007.708634:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.708638:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708640:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205007.708642:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000040:1.0:1788205007.708645:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.708667:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205007.708672:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205007.708672:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708676:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205007.708678:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 02000000:00000001:3.0:1788205007.708679:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205007.708683:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1788205007.708685:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543499264, offset 0 02000000:00000001:3.0:1788205007.708686:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.708690:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205007.708692:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205007.708696:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.708697:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708698:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205007.708702:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205007.708704:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205007.708707:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205007.708709:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.708713:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205007.708714:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205007.708716:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205007.708720:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000400:00000200:1.0:1788205007.708721:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205007.708727:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.708732:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00100000:3.0:1788205007.708736:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205007.708740:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205007.708749:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:3.0:1788205007.708751:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205007.708755:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.708758:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1788205007.708761:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205007.708763:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205007.708772:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471249:471249:256:4294967295] 192.168.201.118@tcp LPNI seq info [471108:471108:8:4294967295] 00000100:00000001:3.0:1788205007.708777:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.708782:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205007.708783:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205007.708788:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205007.708790:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205007.708793:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205007.708799:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205007.708800:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:3.0:1788205007.708806:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26694 00000800:00000200:1.0:1788205007.708813:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205007.708815:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000010:1.0:1788205007.708821:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000400:00000200:3.0:1788205007.708821:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205007.708827:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205007.708831:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.0:1788205007.708835:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205007.708836:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.708841:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.0:1788205007.708843:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:3.0:1788205007.708845:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205007.708850:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205007.708851:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205007.708853:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205007.708856:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000800:00000200:1.1:1788205007.708857:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:3.0:1788205007.708859:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205007.708862:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:1.1:1788205007.708865:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000040:3.0:1788205007.708866:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708869:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.708873:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708877:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000100:1.0:1788205007.708878:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000040:3.0:1788205007.708880:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000800:00000010:1.0:1788205007.708883:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000100:00080000:3.0:1788205007.708884:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205007.708887:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205007.708889:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205007.708891:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205007.708893:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1788205007.708893:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.708897:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205007.708900:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205007.708904:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.708905:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788205007.708906:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205007.708911:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788205007.708914:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000400:00000200:1.0:1788205007.708915:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 02000000:00000001:3.0:1788205007.708916:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205007.708919:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205007.708921:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:3.0:1788205007.708921:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205007.708923:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205007.708925:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.708929:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:3.0:1788205007.708930:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788205007.708934:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205007.708937:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205007.708940:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205007.708969:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205007.708970:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205007.708974:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205007.708978:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205007.708979:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205007.708999:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205007.709002:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205007.709006:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205007.709006:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205007.709009:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205007.709010:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1788205007.709011:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205007.709014:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205007.709016:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26694 00000100:00000001:1.0:1788205007.709020:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205007.709021:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205007.709026:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:1.0:1788205007.709029:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205007.709030:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205007.709032:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321367813 00000100:00000040:3.0:1788205007.709034:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.709038:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.709039:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000010:1.0:1788205007.709040:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647300. 00000100:00000001:3.0:1788205007.709041:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205007.709044:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205007.709045:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205007.709046:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205007.709047:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788205007.709048:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000020:00000001:3.0:1788205007.709049:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205007.709052:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205007.709054:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205007.709056:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205007.709058:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205007.709059:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321367813 00000100:00000001:3.0:1788205007.709060:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205007.709065:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000010:1.0:1788205007.709066:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16479c0. 00000100:00080000:3.0:1788205007.709067:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205007.709070:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205007.709072:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205007.709075:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:3.0:1788205007.709075:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205007.709079:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.709081:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205007.709082:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1788205007.709083:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205007.709088:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.709107:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205007.709109:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205007.709111:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205007.709113:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205007.709114:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.709115:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205007.709116:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205007.709117:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205007.709120:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788205007.709122:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205007.709125:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205007.709129:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205007.709131:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205007.709134:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.709135:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1788205007.709137:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205007.709145:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205007/real 1788205007] req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205007.709145:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205007.709150:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26704) 00000400:00000001:1.0:1788205007.709162:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205007.709165:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205007.709167:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205007.709171:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205007.709173:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205007.709175:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205007.709178:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205007.709187:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709194:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205007.709196:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709199:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.709201:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205007.709205:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709212:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205007.709215:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709223:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205007.709226:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709227:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.709238:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709242:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205007.709244:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205007.709247:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205007.709252:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205007.709254:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205007.709259:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205007.709267:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543499264:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205007.709272:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709274:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.709277:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543499264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709284:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205007.709288:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205007.709291:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:1.0:1788205007.709294:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205007.709297:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205007.709299:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205007.709301:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205007.709304:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205007.709308:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205007.709310:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205007.709311:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205007.709315:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205007.709316:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709318:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709327:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205007.709329:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.709332:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205007.709333:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205007.709335:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205007.709339:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709345:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205007.709347:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.709348:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205007.709351:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205007.709353:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205007.709356:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499392:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205007.709359:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205007.709361:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205007.709362:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.709364:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205007.709367:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba400. 02000000:00000001:1.0:1788205007.709369:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.709373:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205007.709376:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205007.709380:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205007.709383:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543499392, portal 10 00000100:00000001:1.0:1788205007.709385:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205007.709386:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205007.709393:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709400:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205007.709402:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205007.709405:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205007.709407:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543499392, offset 0 00000400:00000200:1.0:1788205007.709412:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205007.709415:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205007.709419:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205007.709422:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205007.709424:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205007.709427:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205007.709429:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205007.709432:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471250:471250:256:4294967295] 192.168.201.118@tcp LPNI seq info [471109:471109:8:4294967295] 00000400:00000200:1.0:1788205007.709436:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205007.709443:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205007.709447:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205007.709450:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205007.709453:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205007.709456:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788205007.709460:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205007.709463:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1788205007.709467:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205007.709468:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205007.709472:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205007.709476:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205007.709479:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205007.709480:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205007.709483:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205007.709486:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709493:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.709496:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709503:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.709504:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205007.709507:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205007.709512:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205007.709515:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205007.709517:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321366547 00000100:00000001:1.0:1788205007.709520:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321367813 00000100:00000001:1.0:1788205007.709522:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321366547 00000100:00000001:1.0:1788205007.709524:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321367813 00000100:00000001:1.0:1788205007.709526:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205007.709527:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.709529:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205007.709533:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205007/real 1788205007] req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205007.709541:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205007.709543:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205007.709544:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205007.709546:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205007.709548:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205007.709550:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205007.709554:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205007.709562:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709568:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205007.709570:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709571:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.709573:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205007.709576:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709582:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205007.709587:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709593:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205007.709595:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709596:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.709598:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709600:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205007.709602:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205007.709604:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205007.709608:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205007.709610:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205007.709614:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205007.709621:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499392:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205007.709625:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205007.709629:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709640:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205007.709641:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.709643:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205007.709645:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205007.709659:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205007.709662:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499520:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205007.709665:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205007.709667:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205007.709669:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.709670:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205007.709674:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205007.709676:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.709679:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205007.709682:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205007.709685:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205007.709688:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543499520, portal 10 00000100:00000001:1.0:1788205007.709690:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205007.709692:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205007.709697:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709703:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205007.709706:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205007.709708:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205007.709711:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543499520, offset 0 00000400:00000200:1.0:1788205007.709715:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205007.709719:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205007.709722:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205007.709726:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205007.709728:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205007.709731:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205007.709733:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205007.709736:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471251:471251:256:4294967295] 192.168.201.118@tcp LPNI seq info [471110:471110:8:4294967295] 00000400:00000200:1.0:1788205007.709740:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205007.709746:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205007.709750:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205007.709754:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205007.709757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205007.709760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788205007.709764:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205007.709768:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1788205007.709771:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205007.709773:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205007.709776:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205007.709779:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205007.709782:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205007.709784:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205007.709786:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205007.709790:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709798:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.709801:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709807:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.709808:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205007.709811:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205007.709814:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205007.709818:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205007.709820:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321366548 00000100:00000001:1.0:1788205007.709822:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321367813 00000100:00000001:1.0:1788205007.709825:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321366548 00000100:00000001:1.0:1788205007.709827:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321367813 00000100:00000001:1.0:1788205007.709829:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205007.709832:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.709833:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205007.709837:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205007/real 1788205007] req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205007.709845:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205007.709846:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205007.709848:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205007.709850:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205007.709852:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205007.709853:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205007.709857:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205007.709865:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709871:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205007.709872:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709874:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.709875:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205007.709878:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709885:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205007.709889:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709895:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205007.709898:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709899:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.709901:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.709903:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205007.709905:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205007.709908:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205007.709911:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205007.709914:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205007.709918:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205007.709926:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499520:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205007.709930:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205007.709933:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709939:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205007.709940:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.709943:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205007.709945:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205007.709946:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205007.709949:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499648:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205007.709952:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205007.709953:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205007.709955:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.709957:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205007.709960:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205007.709962:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.709966:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205007.709968:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205007.709971:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205007.709974:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543499648, portal 4 00000100:00000001:1.0:1788205007.709976:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205007.709977:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788205007.709982:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.709989:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205007.709991:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205007.709994:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205007.709997:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543499648, offset 0 00000400:00000200:1.0:1788205007.710001:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205007.710005:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205007.710009:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205007.710012:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205007.710014:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205007.710016:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205007.710019:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205007.710022:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471252:471252:256:4294967295] 192.168.201.118@tcp LPNI seq info [471111:471111:8:4294967295] 00000400:00000200:1.0:1788205007.710026:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205007.710032:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205007.710037:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205007.710040:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205007.710044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205007.710046:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788205007.710051:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205007.710053:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1788205007.710057:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205007.710059:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205007.710062:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205007.710065:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205007.710068:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205007.710070:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205007.710072:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205007.710075:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710083:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.710085:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710113:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.710115:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205007.710119:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205007.710122:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205007.710125:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205007.710128:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321366558 00000100:00000001:1.0:1788205007.710130:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321367813 00000100:00000001:1.0:1788205007.710132:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321366558 00000100:00000001:1.0:1788205007.710134:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321367813 00000100:00000001:1.0:1788205007.710135:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205007.710138:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.710139:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205007.710143:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205007/real 1788205007] req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205007.710150:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205007.710151:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205007.710153:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205007.710155:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205007.710156:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205007.710158:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205007.710160:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205007.710168:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710174:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205007.710175:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710177:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.710178:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205007.710181:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710187:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205007.710190:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710196:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205007.710199:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710200:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.710201:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710203:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205007.710204:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205007.710207:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205007.710210:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205007.710212:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205007.710215:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205007.710222:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499648:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205007.710226:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205007.710229:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710234:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205007.710235:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.710237:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205007.710239:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205007.710241:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205007.710243:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499776:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205007.710246:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205007.710248:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205007.710249:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205007.710251:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205007.710254:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba000. 02000000:00000001:1.0:1788205007.710256:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205007.710259:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205007.710261:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205007.710264:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205007.710267:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543499776, portal 4 00000100:00000001:1.0:1788205007.710268:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205007.710270:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788205007.710274:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710280:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205007.710282:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205007.710284:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205007.710286:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543499776, offset 0 00000400:00000200:1.0:1788205007.710289:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205007.710293:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205007.710296:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205007.710299:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205007.710301:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205007.710303:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205007.710306:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205007.710308:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471253:471253:256:4294967295] 192.168.201.118@tcp LPNI seq info [471112:471112:8:4294967295] 00000400:00000200:1.0:1788205007.710312:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205007.710317:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205007.710321:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205007.710323:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205007.710327:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205007.710329:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1788205007.710333:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205007.710336:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1788205007.710339:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205007.710341:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205007.710344:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205007.710347:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205007.710350:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205007.710352:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205007.710355:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205007.710358:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710365:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.710368:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205032 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710374:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.710376:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205007.710379:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205007.710382:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205007.710385:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205007.710388:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321366581 00000100:00000001:1.0:1788205007.710390:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321367813 00000100:00000001:1.0:1788205007.710392:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321366581 00000100:00000001:1.0:1788205007.710394:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321367813 00000100:00000001:1.0:1788205007.710396:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205007.710399:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.710400:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205007.710405:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205007/real 1788205007] req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205007.710412:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205007.710414:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205007.710415:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205007.710417:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205007.710419:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205007.710421:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205007.710424:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205007.710432:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710443:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205007.710445:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710447:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.710448:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205007.710452:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710459:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205007.710462:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710469:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205007.710472:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710473:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205007.710474:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710476:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205007.710478:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205007.710481:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205007.710484:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205007.710486:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205007.710490:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205007.710497:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543499776:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205007.710500:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710502:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.710505:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543499392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710512:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205007.710514:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205007.710516:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:1.0:1788205007.710518:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205007.710520:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205007.710521:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205007.710523:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205007.710525:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205007.710528:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205007.710529:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205007.710531:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205007.710533:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205007.710534:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710536:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.710539:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543499520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710546:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205007.710547:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205007.710549:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205007.710551:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205007.710552:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205007.710553:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205007.710555:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205007.710557:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788205007.710559:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205007.710560:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205007.710561:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205007.710563:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205007.710565:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710566:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.710570:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543499648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710576:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205007.710577:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205007.710579:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205007.710581:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205007.710582:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205007.710583:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205007.710586:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205007.710587:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205007.710589:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205007.710590:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205007.710592:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205007.710594:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205007.710595:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710597:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205007.710600:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543499776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205032 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205007.710607:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205007.710608:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205007.710609:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba000. 02000000:00000001:1.0:1788205007.710612:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205007.710613:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205007.710614:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205007.710616:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205007.710617:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205007.710619:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205007.710620:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205007.710621:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205007.710625:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205007.710626:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710628:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205007.710631:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205007.710632:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205007.710634:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205007.710636:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205008.540155:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205008.540155:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205008.540164:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205008.540165:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205008.540167:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205008.540168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205008.540168:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205008.540175:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205008.540175:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205008.540177:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205008.540181:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205008.540182:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205008.540183:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205008.540185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205008.540187:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205008.540188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205008.540189:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205008.540190:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205008.540192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205008.540194:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205008.540196:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788205008.540196:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205008.540197:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205008.540201:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205008.540205:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205008.540208:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205008.540214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205008.540216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205008.540221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205008.540222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205008.540226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205008.540227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205008.540273:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205008.540277:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205008.540280:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205009.052169:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321367813, at 4321367872 00000100:00000040:3.0:1788205009.052169:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321367813, at 4321367872 00000100:00000010:2.0:1788205009.052186:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00000100:00000010:3.0:1788205009.052189:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. 00000100:00000001:3.0:1788205009.564190:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205009.564191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205009.564199:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205009.564201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205009.564202:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205009.564205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205009.564209:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205009.564210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205009.564210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205009.564214:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205009.564216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205009.564217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205009.564219:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205009.564220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205009.564224:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205009.564225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205009.564225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205009.564229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205009.564230:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205009.564231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205009.564235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205009.564238:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205009.564239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205009.564239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205009.564248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205009.564250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205009.564309:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205009.564314:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205009.564317:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205010.588211:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205010.588211:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205010.588216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205010.588220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205010.588221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205010.588222:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205010.588227:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205010.588229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205010.588229:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205010.588231:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205010.588232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205010.588233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205010.588239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205010.588239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205010.588239:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205010.588242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205010.588245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205010.588245:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205010.588246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205010.588249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205010.588249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205010.588250:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205010.588253:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205010.588255:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205010.588259:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205010.588262:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205010.588312:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205010.588316:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205010.588320:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205011.612188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205011.612188:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205011.612200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205011.612200:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205011.612205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.612205:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205011.612214:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205011.612215:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205011.612221:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205011.612222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205011.612228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.612228:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205011.612232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205011.612233:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205011.612237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205011.612238:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205011.612244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205011.612244:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205011.612360:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205011.612366:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205011.612372:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205011.612374:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205011.612377:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205011.612379:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205011.612383:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205011.612384:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205011.612445:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205011.612449:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205011.612452:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205012.636215:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205012.636215:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205012.636221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205012.636224:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205012.636225:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205012.636227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.636230:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.636230:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205012.636236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205012.636238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205012.636239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205012.636239:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205012.636245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205012.636247:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205012.636247:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205012.636249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.636251:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205012.636256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205012.636257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205012.636265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.636265:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205012.636268:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.636279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.636279:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205012.636284:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205012.636288:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205012.636348:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205012.636352:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205012.636355:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205012.828195:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205012.828202:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205012.828204:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205012.828207:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205012.828210:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205012.828216:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26699 00000400:00000200:3.0:1788205012.828224:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205012.828229:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205012.828232:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205012.828234:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828236:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205012.828238:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828239:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205012.828242:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205012.828243:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205012.828245:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205012.828248:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205012.828249:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205012.828251:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205012.828253:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828254:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205012.828255:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828256:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205012.828258:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205012.828260:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828261:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205012.828263:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205012.828266:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205012.828267:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828269:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205012.828271:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205012.828279:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205012.828281:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205012.828284:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205012.828285:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828287:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205012.828290:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205012.828293:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205012.828300:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412c00. 00000100:00000001:3.0:1788205012.828311:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205012.828318:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205012.828326:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205012.828329:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205012.828333:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205012.828356:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205012.828359:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205012.828362:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205012.828364:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205012.828365:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205012.828368:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205012.828371:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26699 00000400:00000200:3.0:1788205012.828374:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205012.828378:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205012.828380:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205012.828383:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828386:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205012.828389:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828391:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205012.828393:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:1.0:1788205012.828394:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1788205012.828395:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205012.828399:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205012.828400:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1788205012.828402:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:1.0:1788205012.828404:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788205012.828408:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205012.828413:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205012.828414:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205012.828421:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828424:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205012.828427:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1788205012.828428:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205012.828430:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828431:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205012.828434:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:1.0:1788205012.828435:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205012.828438:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788205012.828438:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828441:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.828442:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205012.828444:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788205012.828447:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543499904:192.168.201.118@tcp:250:kworker.0 02000000:00000001:3.0:1788205012.828450:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205012.828451:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205012.828455:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.828456:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205012.828457:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205012.828459:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788205012.828461:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788205012.828462:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1788205012.828465:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828467:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205012.828468:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba000. 00000100:00000010:3.0:1788205012.828471:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 02000000:00000001:1.0:1788205012.828472:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205012.828476:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205012.828479:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1788205012.828481:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205012.828481:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828485:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205012.828486:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1788205012.828487:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 02000000:00000001:3.0:1788205012.828488:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205012.828494:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411c00. 00000400:00000010:1.0:1788205012.828495:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205012.828497:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205012.828502:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543499904, portal 25 00000100:00100000:3.0:1788205012.828503:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205012.828507:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205012.828510:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000001:3.0:1788205012.828516:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1788205012.828519:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205012.828519:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205012.828525:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205012.828531:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205012.828536:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205012.828538:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788205012.828541:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:1.0:1788205012.828544:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205012.828547:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205012.828550:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543499904, offset 0 00000100:00000001:3.0:1788205012.828553:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1788205012.828566:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205012.828566:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205012.828571:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205012.828575:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26699 00000400:00000200:1.0:1788205012.828578:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205012.828583:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.828589:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205012.828590:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205012.828595:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205012.828598:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205012.828599:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788205012.828600:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205012.828605:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000040:3.0:1788205012.828606:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828610:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205012.828611:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205012.828613:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:1.0:1788205012.828614:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1788205012.828615:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205012.828618:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000200:1.0:1788205012.828621:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000001:3.0:1788205012.828622:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205012.828626:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205012.828630:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205012.828633:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471254:471254:256:4294967295] 192.168.201.118@tcp LPNI seq info [471113:471113:8:4294967295] 00000100:00000001:3.0:1788205012.828633:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205012.828636:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828640:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205012.828643:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205012.828645:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828647:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205012.828648:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205012.828650:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205012.828651:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205012.828652:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828657:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205012.828659:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205012.828660:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828662:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205012.828664:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205012.828668:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205012.828672:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1788205012.828673:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:3.0:1788205012.828674:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828678:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.0:1788205012.828684:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205012.828686:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205012.828689:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1788205012.828691:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 02000000:00000010:3.0:1788205012.828694:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412000. 00000100:00000001:3.0:1788205012.828699:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205012.828702:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00100000:3.0:1788205012.828708:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1788205012.828709:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205012.828721:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205012.828723:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205012.828727:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788205012.828727:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205012.828737:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000040:3.0:1788205012.828737:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1788205012.828741:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205012.828748:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1788205012.828750:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205012.828754:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1788205012.828756:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205012.828759:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000100:00080000:3.0:1788205012.828762:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:1.0:1788205012.828763:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:3.0:1788205012.828767:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205012.828769:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1788205012.828771:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1788205012.828775:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205012.828777:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205012.828782:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26699 00000100:00000001:1.0:1788205012.828787:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1788205012.828788:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:1.0:1788205012.828792:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205012.828794:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205012.828798:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205012.828804:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205012.828806:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828807:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205012.828810:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1788205012.828811:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828814:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205012.828815:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205012.828816:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000100:00000001:1.0:1788205012.828817:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205012.828818:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205012.828821:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205012.828823:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:1.0:1788205012.828824:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:3.0:1788205012.828825:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205012.828828:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321372933 00000100:00000040:3.0:1788205012.828830:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.828832:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205012.828834:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000010:1.0:1788205012.828835:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647540. 00000100:00000001:3.0:1788205012.828835:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205012.828840:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205012.828842:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205012.828843:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205012.828844:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205012.828847:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205012.828850:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205012.828852:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828853:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205012.828855:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205012.828856:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.828857:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.828858:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321372933 00000100:00000010:3.0:1788205012.828860:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000100:00000010:1.0:1788205012.828863:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16471e0. 00000020:00000040:3.0:1788205012.828864:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788205012.828868:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:3.0:1788205012.828869:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205012.828872:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.828874:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205012.828874:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205012.828876:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205012.828878:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.828880:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205012.828884:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205012/real 1788205012] req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205012.828887:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788205012.828895:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205012.828896:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000010:3.0:1788205012.828896:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000400:00000010:1.0:1788205012.828900:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000100:00000001:3.0:1788205012.828902:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205012.828903:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205012.828906:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205012.828909:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1788205012.828913:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205012.828918:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205012.828929:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205012.828932:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205012.828935:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205012.828938:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.828951:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205012.828954:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205012.828955:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.828957:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1788205012.828958:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205012.828960:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205012.828963:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205012.828965:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788205012.828967:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205012.828968:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205012.828973:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205012.828977:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26699 00000100:00000001:1.0:1788205012.828979:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1788205012.828982:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:1.0:1788205012.828986:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205012.828989:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205012.828992:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205012.828997:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205012.828998:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.828999:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.829002:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205012.829003:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205012.829004:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205012.829006:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205012.829007:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1788205012.829010:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00080000:1.0:1788205012.829011:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788205012.829013:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205012.829017:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00080000:1.0:1788205012.829019:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000020:00000040:3.0:1788205012.829019:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205012.829021:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205012.829022:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:3.0:1788205012.829024:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000040:1.0:1788205012.829028:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205012.829028:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205012.829034:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00100000:1.0:1788205012.829042:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543499904:192.168.201.118@tcp:250:kworker.0 00000100:00000040:3.0:1788205012.829044:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205012.829047:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829050:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205012.829050:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205012.829054:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543499904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205012.829057:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205012.829062:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205012.829064:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205012.829068:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:3.0:1788205012.829068:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 02000000:00000010:1.0:1788205012.829072:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba000. 00000100:00080000:3.0:1788205012.829075:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:1.0:1788205012.829076:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205012.829078:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000001:3.0:1788205012.829080:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205012.829081:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205012.829086:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000001:3.0:1788205012.829086:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1788205012.829111:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:1.0:1788205012.829116:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205012.829117:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205012.829118:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205012.829120:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 02000000:00000001:3.0:1788205012.829123:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829124:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205012.829128:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205012.829128:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205012.829132:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788205012.829133:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205012.829137:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829138:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205012.829140:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829141:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829146:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1788205012.829146:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205012.829148:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1788205012.829151:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788205012.829152:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788205012.829157:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1788205012.829159:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205012.829160:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205012.829163:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205012.829164:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205012.829167:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205012.829172:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:1.0:1788205012.829174:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205012.829176:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829178:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829182:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1788205012.829185:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205012.829186:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205012.829191:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205012.829200:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500032:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205012.829202:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205012.829204:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788205012.829207:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829208:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205012.829210:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829212:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205012.829216:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba000. 00000100:00000040:3.0:1788205012.829216:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205012.829220:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205012.829226:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205012.829230:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000100:00000001:3.0:1788205012.829234:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205012.829235:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205012.829239:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543500032, portal 10 00000100:00000001:1.0:1788205012.829242:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205012.829244:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:3.0:1788205012.829244:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26709) 00000100:00000040:1.0:1788205012.829253:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829260:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205012.829263:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205012.829265:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205012.829268:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543500032, offset 0 00000400:00000200:1.0:1788205012.829273:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205012.829277:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.829282:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205012.829285:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205012.829287:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205012.829289:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205012.829292:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205012.829295:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471255:471255:256:4294967295] 192.168.201.118@tcp LPNI seq info [471114:471114:8:4294967295] 00000400:00000200:1.0:1788205012.829298:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.829304:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205012.829308:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205012.829311:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205012.829315:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205012.829318:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205012.829322:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205012.829324:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205012.829328:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205012.829329:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205012.829332:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205012.829335:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205012.829338:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205012.829339:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205012.829342:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205012.829345:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829352:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205012.829354:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829360:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829361:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205012.829364:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205012.829368:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205012.829370:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205012.829372:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321371667 00000100:00000001:1.0:1788205012.829375:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321372933 00000100:00000001:1.0:1788205012.829376:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321371667 00000100:00000001:1.0:1788205012.829378:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321372933 00000100:00000001:1.0:1788205012.829380:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205012.829382:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829384:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205012.829387:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205012/real 1788205012] req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205012.829394:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205012.829395:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205012.829397:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205012.829399:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205012.829401:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205012.829403:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205012.829406:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205012.829414:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829420:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205012.829421:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829423:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.829424:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205012.829428:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829434:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205012.829437:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829443:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205012.829446:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829447:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.829448:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829450:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205012.829452:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205012.829454:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205012.829457:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205012.829459:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205012.829463:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205012.829470:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500032:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205012.829474:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205012.829477:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829482:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205012.829484:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829486:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205012.829487:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205012.829489:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205012.829492:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500160:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205012.829495:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205012.829496:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205012.829498:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829499:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205012.829502:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205012.829504:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205012.829508:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205012.829510:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205012.829512:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205012.829515:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543500160, portal 10 00000100:00000001:1.0:1788205012.829516:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205012.829517:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205012.829522:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829528:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205012.829530:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205012.829532:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205012.829534:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543500160, offset 0 00000400:00000200:1.0:1788205012.829537:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205012.829541:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.829544:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205012.829547:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205012.829549:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205012.829551:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205012.829553:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205012.829556:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471256:471256:256:4294967295] 192.168.201.118@tcp LPNI seq info [471115:471115:8:4294967295] 00000400:00000200:1.0:1788205012.829573:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.829580:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205012.829584:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205012.829588:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205012.829591:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205012.829594:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205012.829598:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205012.829600:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205012.829604:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205012.829606:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205012.829609:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205012.829612:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205012.829614:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205012.829616:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205012.829618:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205012.829621:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829629:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205012.829632:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829637:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829639:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205012.829642:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205012.829645:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205012.829648:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205012.829650:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321371668 00000100:00000001:1.0:1788205012.829652:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321372933 00000100:00000001:1.0:1788205012.829653:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321371668 00000100:00000001:1.0:1788205012.829655:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321372933 00000100:00000001:1.0:1788205012.829656:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205012.829658:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829660:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205012.829663:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205012/real 1788205012] req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205012.829669:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205012.829670:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205012.829671:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205012.829673:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205012.829674:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205012.829676:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205012.829678:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205012.829684:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829689:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205012.829690:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829692:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.829693:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205012.829696:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829701:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205012.829704:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829710:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205012.829712:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829713:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.829714:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829716:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205012.829718:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205012.829720:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205012.829723:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205012.829725:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205012.829729:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205012.829737:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500160:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205012.829741:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205012.829744:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829751:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205012.829752:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829754:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205012.829755:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205012.829758:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205012.829760:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500288:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205012.829763:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205012.829764:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205012.829765:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829766:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205012.829769:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205012.829770:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205012.829772:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205012.829774:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205012.829776:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205012.829778:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543500288, portal 4 00000100:00000001:1.0:1788205012.829780:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205012.829780:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205012.829784:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829788:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205012.829790:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205012.829791:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205012.829792:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543500288, offset 0 00000400:00000200:1.0:1788205012.829794:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205012.829797:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.829799:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205012.829801:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205012.829802:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205012.829804:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205012.829805:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205012.829807:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471257:471257:256:4294967295] 192.168.201.118@tcp LPNI seq info [471116:471116:8:4294967295] 00000400:00000200:1.0:1788205012.829809:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.829813:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205012.829816:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205012.829818:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205012.829820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205012.829822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205012.829824:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205012.829826:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205012.829828:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205012.829830:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205012.829831:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205012.829834:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205012.829835:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205012.829836:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205012.829838:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205012.829839:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829848:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205012.829849:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829853:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829854:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205012.829855:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205012.829857:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205012.829859:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205012.829860:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321371677 00000100:00000001:1.0:1788205012.829861:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321372933 00000100:00000001:1.0:1788205012.829862:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321371677 00000100:00000001:1.0:1788205012.829863:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321372933 00000100:00000001:1.0:1788205012.829865:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205012.829866:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829867:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205012.829869:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205012/real 1788205012] req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205012.829873:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205012.829873:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205012.829874:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205012.829876:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205012.829876:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205012.829878:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205012.829879:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205012.829884:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829887:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205012.829888:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829889:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.829890:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205012.829892:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829895:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205012.829897:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829901:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205012.829902:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829903:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.829904:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829904:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205012.829906:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205012.829907:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205012.829909:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205012.829910:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205012.829912:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205012.829916:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500288:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205012.829917:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829919:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205012.829920:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543500032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829924:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205012.829925:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205012.829926:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba000. 02000000:00000001:1.0:1788205012.829928:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205012.829929:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205012.829930:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205012.829931:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205012.829932:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788205012.829934:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205012.829935:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205012.829935:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205012.829937:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205012.829938:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829939:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205012.829941:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543500160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829944:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205012.829945:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205012.829946:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205012.829947:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205012.829948:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205012.829948:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205012.829949:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205012.829950:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788205012.829951:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205012.829952:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205012.829952:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205012.829953:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205012.829954:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829955:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205012.829956:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543500288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829959:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205012.829960:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205012.829961:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205012.829962:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205012.829962:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205012.829963:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205012.829964:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205012.829964:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205012.829965:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205012.829966:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205012.829967:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205012.829968:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205012.829969:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829970:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.829973:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205012.829974:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.829975:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205012.829976:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205012.829977:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205012.829978:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.829982:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205012.829983:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829985:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205012.829986:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205012.829987:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205012.829988:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500416:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205012.829990:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205012.829991:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205012.829991:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205012.829992:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205012.829994:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205012.829995:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205012.829996:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205012.829998:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205012.829999:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205012.830001:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543500416, portal 4 00000100:00000001:1.0:1788205012.830002:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205012.830003:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205012.830006:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.830009:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205012.830011:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205012.830012:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205012.830013:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543500416, offset 0 00000400:00000200:1.0:1788205012.830015:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205012.830018:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.830019:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205012.830021:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205012.830023:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205012.830024:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205012.830026:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205012.830027:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471258:471258:256:4294967295] 192.168.201.118@tcp LPNI seq info [471117:471117:8:4294967295] 00000400:00000200:1.0:1788205012.830030:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205012.830033:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205012.830036:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205012.830037:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205012.830039:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205012.830041:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205012.830043:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205012.830045:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205012.830047:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205012.830048:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205012.830049:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205012.830051:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205012.830053:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205012.830054:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205012.830055:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205012.830057:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.830061:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205012.830062:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205037 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.830066:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.830067:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205012.830069:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205012.830070:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205012.830072:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205012.830073:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321371700 00000100:00000001:1.0:1788205012.830075:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321372933 00000100:00000001:1.0:1788205012.830076:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321371700 00000100:00000001:1.0:1788205012.830077:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321372933 00000100:00000001:1.0:1788205012.830078:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205012.830080:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.830081:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205012.830083:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205012/real 1788205012] req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205012.830086:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205012.830087:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205012.830088:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205012.830104:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205012.830105:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205012.830107:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205012.830109:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205012.830113:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.830117:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205012.830118:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.830119:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.830119:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205012.830121:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.830124:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205012.830126:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.830131:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205012.830132:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.830133:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205012.830134:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.830135:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205012.830136:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205012.830137:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205012.830139:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205012.830140:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205012.830142:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205012.830146:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500416:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205012.830148:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.830149:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205012.830151:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543500416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205037 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205012.830156:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205012.830158:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205012.830159:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205012.830161:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205012.830163:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205012.830164:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205012.830166:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205012.830167:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205012.830169:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205012.830170:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205012.830171:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205012.830173:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205012.830175:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.830177:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205012.830180:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205012.830181:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205012.830183:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205012.830184:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205013.660177:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205013.660177:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205013.660180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205013.660184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205013.660185:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205013.660185:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205013.660188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205013.660190:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205013.660192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205013.660194:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205013.660196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205013.660197:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205013.660200:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205013.660201:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205013.660203:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205013.660205:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205013.660205:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205013.660206:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205013.660208:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205013.660211:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205013.660211:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205013.660211:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205013.660216:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205013.660218:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788205013.660220:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660221:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660225:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660228:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205013.660231:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205013.660232:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205013.660234:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660235:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205013.660236:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660237:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660239:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205013.660242:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205013.660243:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205013.660243:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660244:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205013.660245:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660246:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205013.660247:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205013.660291:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205013.660294:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205013.660296:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205014.172185:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321372933, at 4321372992 00000100:00000040:3.0:1788205014.172185:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321372933, at 4321372992 00000100:00000010:3.0:1788205014.172200:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00000100:00000010:2.0:1788205014.172201:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647540. 00000100:00000001:1.0:1788205014.684180:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205014.684180:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205014.684184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205014.684187:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205014.684187:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205014.684189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205014.684190:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205014.684191:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205014.684196:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205014.684197:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205014.684201:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205014.684202:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205014.684206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205014.684206:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205014.684209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205014.684212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205014.684212:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205014.684213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205014.684217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205014.684217:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205014.684219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205014.684223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205014.684223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205014.684224:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205014.684228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205014.684231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205014.684283:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205014.684286:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205014.684289:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205015.708195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205015.708199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205015.708204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205015.708207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205015.708212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205015.708212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205015.708217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205015.708222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205015.708225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205015.708227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205015.708227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205015.708230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205015.708233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205015.708236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205015.708239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205015.708243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205015.708244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205015.708296:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205015.708300:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205015.708308:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205015.708323:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205015.708328:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205015.708329:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205015.708332:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205015.708334:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205015.708336:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205015.708337:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205015.708339:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205015.708341:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.732188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205016.732188:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205016.732194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205016.732196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205016.732196:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205016.732199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.732199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.732200:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205016.732206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205016.732207:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205016.732208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205016.732212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.732212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205016.732213:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205016.732216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205016.732217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205016.732217:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205016.732219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205016.732222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205016.732224:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205016.732225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205016.732225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205016.732228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.732229:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205016.732230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205016.732233:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205016.732290:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205016.732295:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205016.732298:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205017.756187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205017.756187:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205017.756190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205017.756194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205017.756195:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205017.756196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.756198:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205017.756201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205017.756203:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205017.756205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205017.756209:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.756209:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205017.756211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.756214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205017.756214:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205017.756216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205017.756217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205017.756220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.756220:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205017.756221:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205017.756226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205017.756226:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205017.756228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.756231:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205017.756232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205017.756234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205017.756284:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205017.756288:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205017.756290:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205017.948226:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205017.948236:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205017.948238:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205017.948242:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205017.948245:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205017.948252:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26704 00000400:00000200:1.0:1788205017.948260:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205017.948267:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205017.948270:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205017.948273:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948275:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205017.948278:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948280:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205017.948284:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205017.948286:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205017.948289:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205017.948292:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205017.948295:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205017.948297:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205017.948300:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948302:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205017.948304:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948306:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205017.948308:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205017.948312:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948314:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205017.948317:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205017.948320:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205017.948323:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948325:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205017.948327:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205017.948336:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205017.948339:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205017.948342:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205017.948344:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948347:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205017.948350:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205017.948353:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205017.948360:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9000. 00000100:00000001:1.0:1788205017.948366:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205017.948374:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205017.948383:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205017.948388:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205017.948392:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205017.948413:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205017.948416:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205017.948419:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205017.948421:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205017.948422:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205017.948425:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205017.948429:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26704 00000400:00000200:1.0:1788205017.948432:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205017.948436:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205017.948439:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205017.948440:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1788205017.948442:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205017.948444:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205017.948446:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205017.948447:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205017.948451:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948455:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788205017.948456:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205017.948460:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205017.948463:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205017.948466:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788205017.948467:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205017.948479:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205017.948482:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 02000000:00000001:3.0:1788205017.948483:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205017.948486:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788205017.948487:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 02000000:00000001:3.0:1788205017.948489:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205017.948492:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00100000:3.0:1788205017.948494:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543500544:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205017.948499:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1788205017.948500:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948504:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788205017.948504:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205017.948507:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205017.948508:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:3.0:1788205017.948510:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788205017.948513:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205017.948517:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000010:3.0:1788205017.948518:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 00000100:00080000:1.0:1788205017.948520:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788205017.948523:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948524:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205017.948528:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205017.948531:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:1.0:1788205017.948533:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1788205017.948536:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 02000000:00000001:1.0:1788205017.948537:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948541:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205017.948546:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000010:3.0:1788205017.948548:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000008:00000001:1.0:1788205017.948550:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948553:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205017.948556:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543500544, portal 25 00000100:00000010:1.0:1788205017.948558:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205017.948560:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:1.0:1788205017.948562:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000001:3.0:1788205017.948563:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 02000000:00000001:1.0:1788205017.948569:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205017.948573:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205017.948573:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205017.948576:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205017.948579:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205017.948582:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.948586:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:1.0:1788205017.948588:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba000. 00000100:00000040:3.0:1788205017.948589:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205017.948593:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000001:1.0:1788205017.948594:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205017.948596:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543500544, offset 0 00000400:00000200:3.0:1788205017.948602:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00100000:1.0:1788205017.948603:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205017.948611:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205017.948617:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788205017.948618:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:1.0:1788205017.948621:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205017.948624:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1788205017.948628:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205017.948630:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205017.948634:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205017.948638:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205017.948645:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471259:471259:256:4294967295] 192.168.201.118@tcp LPNI seq info [471118:471118:8:4294967295] 00000100:00000001:1.0:1788205017.948647:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205017.948649:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:1.0:1788205017.948654:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205017.948662:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:3.0:1788205017.948662:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:1.0:1788205017.948666:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205017.948669:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.0:1788205017.948670:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948673:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:3.0:1788205017.948675:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000100:00080000:1.0:1788205017.948679:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26704 00000800:00000200:3.0:1788205017.948683:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:1.0:1788205017.948687:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.0:1788205017.948687:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000400:00000200:1.0:1788205017.948694:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:3.1:1788205017.948695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205017.948698:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205017.948701:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205017.948702:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000100:00080000:1.0:1788205017.948706:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205017.948709:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205017.948716:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1788205017.948717:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948721:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205017.948721:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205017.948724:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000040:1.0:1788205017.948726:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205017.948729:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1788205017.948730:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000200:3.0:1788205017.948734:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205017.948735:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205017.948738:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205017.948742:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205017.948746:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000001:3.0:1788205017.948747:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1788205017.948751:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205017.948752:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205017.948758:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205017.948767:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205017.948769:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948771:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205017.948771:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1788205017.948776:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948779:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205017.948781:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205017.948784:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205017.948785:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948787:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205017.948791:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205017.948791:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205017.948794:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321378053 02000000:00000001:1.0:1788205017.948796:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1788205017.948799:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fccc0. 02000000:00000001:1.0:1788205017.948800:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948804:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205017.948806:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788205017.948806:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000008:00000001:1.0:1788205017.948811:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.948812:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321378053 00000100:00000001:1.0:1788205017.948814:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205017.948816:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc660. 00000100:00000001:3.0:1788205017.948819:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205017.948821:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1788205017.948822:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205017.948825:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205017.948827:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000100:00000001:3.0:1788205017.948828:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788205017.948833:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788205017.948833:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205017.948837:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.948841:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205017.948843:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:3.0:1788205017.948845:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1788205017.948846:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788205017.948849:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000010:1.0:1788205017.948853:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba400. 00000400:00000010:3.0:1788205017.948853:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205017.948858:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000100:00000001:1.0:1788205017.948860:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205017.948862:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205017.948866:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1788205017.948870:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788205017.948873:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205017.948894:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788205017.948897:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205017.948898:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205017.948907:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.948909:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205017.948912:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.948916:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.948919:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.948922:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205017.948927:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205017.948928:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205017.948936:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205017.948940:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205017.948943:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205017.948944:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.948950:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:3.0:1788205017.948952:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205017.948956:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26704 00000400:00000200:1.0:1788205017.948965:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205017.948965:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205017.948971:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1788205017.948972:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205017.948974:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.948977:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788205017.948978:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205017.948980:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:1.0:1788205017.948984:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205017.948984:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205017.948988:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:3.0:1788205017.948990:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1788205017.948994:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.948999:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00080000:3.0:1788205017.948999:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000020:00000001:1.0:1788205017.949004:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:3.0:1788205017.949005:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:1.0:1788205017.949008:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205017.949012:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:3.0:1788205017.949012:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205017.949016:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205017.949021:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205017.949026:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00100000:3.0:1788205017.949028:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543500544:192.168.201.118@tcp:250:kworker.0 00000100:00000040:1.0:1788205017.949033:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205017.949035:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.949039:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205017.949039:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205017.949046:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205017.949046:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543500544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205017.949052:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205017.949054:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205017.949057:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:1.0:1788205017.949058:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 02000000:00000010:3.0:1788205017.949059:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205017.949065:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1788205017.949067:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000020:00000001:3.0:1788205017.949067:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205017.949070:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000100:00000001:1.0:1788205017.949074:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205017.949074:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205017.949077:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205017.949078:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205017.949080:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205017.949083:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205017.949083:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205017.949085:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 02000000:00000001:1.0:1788205017.949088:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949116:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205017.949117:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949118:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.949124:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205017.949128:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:3.0:1788205017.949128:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205017.949131:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788205017.949133:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949134:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205017.949136:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205017.949138:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949139:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205017.949143:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1788205017.949146:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205017.949151:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1788205017.949152:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788205017.949153:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.949155:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205017.949158:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205017.949159:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205017.949160:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205017.949163:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205017.949164:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500672:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205017.949168:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205017.949169:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205017.949172:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205017.949173:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205017.949173:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.949176:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205017.949177:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205017.949178:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205017.949181:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205017.949184:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9000. 00000400:00000200:3.0:1788205017.949184:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205017.949188:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:1.0:1788205017.949190:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205017.949192:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205017.949196:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543500672, portal 10 00000100:00000001:3.0:1788205017.949198:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205017.949200:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00100000:1.0:1788205017.949201:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788205017.949207:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949224:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788205017.949225:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788205017.949227:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788205017.949230:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205017.949231:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205017.949234:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543500672, offset 0 00000400:00000200:3.0:1788205017.949239:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000040:1.0:1788205017.949240:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205017.949243:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205017.949248:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205017.949252:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205017.949256:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205017.949260:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205017.949262:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:1.0:1788205017.949264:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205017.949265:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471260:471260:256:4294967295] 192.168.201.118@tcp LPNI seq info [471119:471119:8:4294967295] 00000100:00080000:1.0:1788205017.949269:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205017.949270:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:1.0:1788205017.949277:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1788205017.949279:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:1.0:1788205017.949281:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.0:1788205017.949284:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:1.0:1788205017.949285:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:3.0:1788205017.949287:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000100:00000001:1.0:1788205017.949292:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.0:1788205017.949293:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:1.0:1788205017.949298:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26704 00000800:00000200:3.0:1788205017.949301:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000400:00000200:1.0:1788205017.949309:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.1:1788205017.949309:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205017.949314:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000400:00000200:1.0:1788205017.949319:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000100:3.0:1788205017.949319:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205017.949322:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000100:00080000:1.0:1788205017.949325:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205017.949326:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205017.949333:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1788205017.949335:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205017.949336:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205017.949339:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:1.0:1788205017.949341:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205017.949342:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1788205017.949348:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000200:3.0:1788205017.949348:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205017.949355:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205017.949358:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1788205017.949361:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000040:3.0:1788205017.949362:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205017.949365:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000001:3.0:1788205017.949368:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205017.949370:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788205017.949371:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:1.0:1788205017.949375:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:3.0:1788205017.949375:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205017.949379:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000020:00000001:1.0:1788205017.949380:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000200:3.0:1788205017.949384:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:1.0:1788205017.949386:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205017.949387:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321376787 00000100:00000001:3.0:1788205017.949390:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321378053 00000100:00000001:3.0:1788205017.949393:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321376787 00000100:00000040:1.0:1788205017.949395:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205017.949395:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321378053 00000100:00000001:3.0:1788205017.949399:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205017.949401:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205017.949402:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949404:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1788205017.949407:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00100000:3.0:1788205017.949409:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205017.949412:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205017.949419:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000001:3.0:1788205017.949420:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205017.949422:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205017.949424:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00080000:1.0:1788205017.949427:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:3.0:1788205017.949427:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205017.949429:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205017.949432:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1788205017.949434:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205017.949436:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205017.949439:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:3.0:1788205017.949444:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205017.949445:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205017.949451:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205017.949454:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205017.949454:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205017.949458:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1788205017.949459:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1788205017.949462:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:1.0:1788205017.949464:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949466:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.949467:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205017.949487:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1788205017.949488:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205017.949493:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205017.949499:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205017.949502:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949503:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.949506:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205017.949509:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:3.0:1788205017.949510:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205017.949511:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205017.949519:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9800. 00000100:00000001:3.0:1788205017.949520:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205017.949524:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205017.949525:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949525:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.949527:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949530:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205017.949532:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205017.949534:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788205017.949536:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788205017.949538:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205017.949541:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205017.949547:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205017.949562:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500672:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205017.949569:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205017.949572:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205017.949575:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205017.949579:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205017.949580:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949581:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.949585:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205017.949587:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205017.949590:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205017.949590:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205017.949593:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500800:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205017.949598:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205017.949600:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205017.949601:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.949604:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205017.949608:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:1.0:1788205017.949610:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.949610:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205017.949616:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:1.0:1788205017.949618:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26714) 00000400:00000010:3.0:1788205017.949618:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205017.949625:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205017.949627:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543500800, portal 10 00000100:00000001:3.0:1788205017.949628:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205017.949629:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205017.949634:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949638:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205017.949641:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205017.949642:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205017.949644:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543500800, offset 0 00000400:00000200:3.0:1788205017.949646:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205017.949649:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205017.949651:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205017.949653:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205017.949655:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205017.949656:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205017.949658:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205017.949661:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471261:471261:256:4294967295] 192.168.201.118@tcp LPNI seq info [471120:471120:8:4294967295] 00000400:00000200:3.0:1788205017.949665:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205017.949670:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205017.949674:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205017.949677:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205017.949681:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205017.949684:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205017.949688:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205017.949691:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205017.949695:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205017.949697:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205017.949700:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205017.949703:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205017.949706:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205017.949709:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205017.949711:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205017.949715:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949723:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205017.949726:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949733:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949735:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205017.949738:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205017.949740:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205017.949743:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205017.949745:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321376788 00000100:00000001:3.0:1788205017.949748:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321378053 00000100:00000001:3.0:1788205017.949750:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321376788 00000100:00000001:3.0:1788205017.949752:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321378053 00000100:00000001:3.0:1788205017.949754:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205017.949756:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949758:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205017.949762:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205017.949770:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205017.949771:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205017.949773:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205017.949775:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205017.949777:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205017.949779:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205017.949782:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205017.949790:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949798:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205017.949799:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949801:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.949802:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205017.949806:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949812:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205017.949816:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949823:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205017.949826:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949827:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.949829:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949830:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205017.949833:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205017.949835:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205017.949838:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205017.949841:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205017.949844:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205017.949852:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500800:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205017.949856:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949857:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205017.949861:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543500672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949867:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205017.949869:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205017.949871:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205017.949874:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205017.949875:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205017.949877:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205017.949879:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205017.949880:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba000. 02000000:00000001:3.0:1788205017.949883:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205017.949884:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205017.949885:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205017.949891:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205017.949892:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949894:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205017.949897:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543500800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949903:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205017.949904:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205017.949905:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205017.949908:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205017.949909:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205017.949910:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205017.949912:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205017.949913:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:3.0:1788205017.949915:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205017.949916:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205017.949917:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205017.949919:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205017.949920:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949922:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.949926:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205017.949927:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.949929:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205017.949930:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205017.949932:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205017.949935:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949942:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205017.949943:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.949945:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205017.949947:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205017.949949:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205017.949952:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500928:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205017.949955:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205017.949957:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205017.949958:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.949960:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205017.949963:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205017.949965:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205017.949969:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205017.949971:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205017.949974:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205017.949977:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543500928, portal 4 00000100:00000001:3.0:1788205017.949979:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205017.949980:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788205017.949985:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.949992:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205017.949994:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205017.949996:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205017.949998:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543500928, offset 0 00000400:00000200:3.0:1788205017.950002:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205017.950006:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205017.950009:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205017.950012:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205017.950014:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205017.950017:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205017.950019:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205017.950022:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471262:471262:256:4294967295] 192.168.201.118@tcp LPNI seq info [471121:471121:8:4294967295] 00000400:00000200:3.0:1788205017.950026:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205017.950032:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205017.950037:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205017.950040:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205017.950044:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205017.950046:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205017.950050:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205017.950053:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205017.950056:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205017.950058:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205017.950060:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205017.950064:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205017.950067:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205017.950068:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205017.950071:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205017.950075:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950082:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205017.950084:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950108:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.950110:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205017.950113:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205017.950115:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205017.950119:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205017.950121:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321376798 00000100:00000001:3.0:1788205017.950123:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321378053 00000100:00000001:3.0:1788205017.950126:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321376798 00000100:00000001:3.0:1788205017.950128:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321378053 00000100:00000001:3.0:1788205017.950130:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205017.950132:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.950134:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205017.950138:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205017.950184:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205017.950186:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205017.950188:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205017.950190:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205017.950192:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205017.950194:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205017.950198:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205017.950206:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950212:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205017.950213:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950215:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.950216:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205017.950220:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950227:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205017.950232:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950239:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205017.950242:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950245:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.950246:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950248:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205017.950250:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205017.950253:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205017.950257:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205017.950259:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205017.950263:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205017.950270:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543500928:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205017.950274:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205017.950276:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950282:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205017.950283:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.950285:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205017.950287:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205017.950289:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205017.950292:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501056:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205017.950295:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205017.950296:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205017.950298:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205017.950299:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205017.950302:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205017.950304:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205017.950308:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205017.950311:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205017.950314:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205017.950318:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543501056, portal 4 00000100:00000001:3.0:1788205017.950320:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205017.950321:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205017.950326:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950332:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205017.950334:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205017.950336:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205017.950338:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543501056, offset 0 00000400:00000200:3.0:1788205017.950343:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205017.950347:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205017.950350:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205017.950353:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205017.950355:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205017.950358:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205017.950360:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205017.950363:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471263:471263:256:4294967295] 192.168.201.118@tcp LPNI seq info [471122:471122:8:4294967295] 00000400:00000200:3.0:1788205017.950368:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205017.950374:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205017.950378:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205017.950382:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205017.950385:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205017.950388:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205017.950406:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205017.950409:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205017.950413:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205017.950415:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205017.950418:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205017.950422:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205017.950425:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205017.950427:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205017.950429:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205017.950433:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950440:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205017.950443:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205042 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950449:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.950451:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205017.950453:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205017.950456:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205017.950459:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205017.950461:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321376821 00000100:00000001:3.0:1788205017.950464:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321378053 00000100:00000001:3.0:1788205017.950466:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321376821 00000100:00000001:3.0:1788205017.950468:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321378053 00000100:00000001:3.0:1788205017.950481:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205017.950483:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.950485:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205017.950489:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205017/real 1788205017] req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205017.950497:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205017.950499:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205017.950500:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205017.950503:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205017.950505:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205017.950507:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205017.950510:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205017.950519:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950525:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205017.950527:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950529:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.950530:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205017.950533:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950540:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205017.950544:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950550:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205017.950553:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950555:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205017.950556:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950557:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205017.950559:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205017.950562:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205017.950565:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205017.950567:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205017.950571:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205017.950579:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501056:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205017.950582:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950585:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205017.950588:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543500928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950594:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205017.950596:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205017.950598:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205017.950600:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205017.950602:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205017.950603:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205017.950605:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205017.950607:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:3.0:1788205017.950609:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205017.950610:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205017.950611:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205017.950614:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205017.950616:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950617:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205017.950620:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543501056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205042 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205017.950633:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205017.950634:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205017.950635:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205017.950637:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205017.950639:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205017.950640:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205017.950642:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205017.950643:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:3.0:1788205017.950645:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205017.950646:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205017.950648:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205017.950649:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205017.950651:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950652:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205017.950656:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205017.950657:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205017.950659:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205017.950660:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205018.780183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205018.780183:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205018.780187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205018.780190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205018.780190:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205018.780192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205018.780193:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205018.780194:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205018.780197:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:1.0:1788205018.780199:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205018.780200:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205018.780202:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205018.780203:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205018.780207:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788205018.780207:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205018.780211:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205018.780211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205018.780213:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205018.780215:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205018.780215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205018.780216:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205018.780221:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205018.780222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205018.780223:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205018.780224:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205018.780228:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205018.780232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205018.780235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205018.780239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205018.780241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205018.780244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205018.780246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205018.780294:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205018.780299:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205018.780302:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205019.292187:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321378053, at 4321378112 00000100:00000040:3.0:1788205019.292187:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321378053, at 4321378112 00000100:00000010:2.0:1788205019.292202:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fccc0. 00000100:00000010:3.0:1788205019.292204:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc660. 00000100:00000001:1.0:1788205019.804160:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205019.804162:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205019.804166:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205019.804169:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205019.804169:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205019.804169:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205019.804173:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205019.804174:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205019.804175:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205019.804180:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205019.804181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205019.804181:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205019.804184:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205019.804185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205019.804186:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205019.804188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205019.804191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205019.804192:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205019.804192:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205019.804194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205019.804197:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205019.804199:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205019.804200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205019.804203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205019.804204:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205019.804208:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205019.804251:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205019.804253:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205019.804256:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205020.828184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205020.828184:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205020.828190:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205020.828191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205020.828192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205020.828193:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205020.828196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205020.828197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205020.828204:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205020.828205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205020.828206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205020.828211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205020.828212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205020.828212:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205020.828216:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205020.828217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205020.828218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205020.828220:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205020.828222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205020.828223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205020.828223:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205020.828226:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205020.828230:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205020.828231:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205020.828232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205020.828236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205020.828284:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205020.828287:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205020.828289:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205021.852153:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205021.852153:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205021.852153:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205021.852160:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205021.852161:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205021.852161:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205021.852164:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.852164:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205021.852168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205021.852170:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205021.852170:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205021.852171:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205021.852175:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205021.852175:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205021.852178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205021.852179:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205021.852181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.852184:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205021.852187:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205021.852188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205021.852189:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205021.852189:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205021.852191:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205021.852193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205021.852194:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205021.852195:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205021.852274:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205021.852277:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205021.852279:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205022.876179:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205022.876184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205022.876185:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205022.876188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205022.876188:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205022.876195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205022.876195:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205022.876197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205022.876198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205022.876199:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205022.876201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205022.876203:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205022.876204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205022.876205:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205022.876208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205022.876209:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205022.876211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205022.876211:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1788205022.876214:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205022.876216:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205022.876219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205022.876220:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205022.876224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205022.876228:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205022.876230:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205022.876233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205022.876247:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205022.876250:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205022.876252:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205023.068190:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205023.068197:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205023.068199:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205023.068202:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205023.068205:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205023.068211:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26709 00000400:00000200:1.0:1788205023.068218:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205023.068223:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205023.068226:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205023.068227:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068230:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068231:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068233:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205023.068236:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205023.068238:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205023.068240:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205023.068242:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205023.068244:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205023.068246:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205023.068247:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068249:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068250:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068251:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205023.068253:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205023.068255:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068256:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205023.068258:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205023.068260:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205023.068262:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068263:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205023.068265:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205023.068273:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205023.068276:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205023.068279:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205023.068280:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068282:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205023.068284:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205023.068287:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205023.068292:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9c00. 00000100:00000001:1.0:1788205023.068298:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205023.068304:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205023.068311:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205023.068314:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205023.068317:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205023.068335:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205023.068337:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205023.068339:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205023.068341:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205023.068342:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205023.068344:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205023.068345:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26709 00000400:00000200:1.0:1788205023.068348:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205023.068349:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205023.068351:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205023.068352:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068354:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068355:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068356:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205023.068357:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205023.068358:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205023.068359:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205023.068361:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205023.068362:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205023.068363:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205023.068365:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068367:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068368:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068369:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205023.068370:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205023.068372:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068373:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205023.068375:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205023.068379:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205023.068391:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068391:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205023.068393:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205023.068398:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788205023.068398:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000008:00000001:1.0:1788205023.068402:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068402:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205023.068404:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205023.068407:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000020:00000040:1.0:1788205023.068411:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000040:3.0:1788205023.068412:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205023.068413:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205023.068415:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068418:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205023.068419:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205023.068421:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205023.068426:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb800. 00000100:00000001:1.0:1788205023.068430:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068431:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205023.068435:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205023.068437:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788205023.068441:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788205023.068445:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788205023.068446:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205023.068447:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.068451:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205023.068452:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205023.068459:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543501184:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205023.068461:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205023.068465:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205023.068468:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205023.068468:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788205023.068471:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205023.068472:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1788205023.068474:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788205023.068475:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205023.068479:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26709 02000000:00000001:3.0:1788205023.068479:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205023.068484:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205023.068485:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1788205023.068486:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205023.068488:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205023.068490:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000010:3.0:1788205023.068491:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:1.0:1788205023.068492:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068495:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068497:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:3.0:1788205023.068497:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205023.068502:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205023.068503:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205023.068504:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205023.068506:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000200:3.0:1788205023.068507:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:1.0:1788205023.068508:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205023.068512:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205023.068515:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000010:3.0:1788205023.068515:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:1.0:1788205023.068517:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068520:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068522:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205023.068523:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205023.068525:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:3.0:1788205023.068526:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000001:1.0:1788205023.068527:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205023.068530:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205023.068533:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205023.068535:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068536:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:3.0:1788205023.068537:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543501184, portal 25 00000008:00100000:1.0:1788205023.068538:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205023.068543:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068543:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205023.068547:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068547:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000010:1.0:1788205023.068554:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000100:00000040:3.0:1788205023.068557:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205023.068558:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205023.068564:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205023.068567:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068571:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205023.068574:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205023.068574:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1788205023.068578:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205023.068581:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000010:1.0:1788205023.068584:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bbc00. 00000100:00000001:1.0:1788205023.068588:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205023.068591:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00100000:1.0:1788205023.068595:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788205023.068597:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543501184, offset 0 00000100:00000001:1.0:1788205023.068605:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205023.068608:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205023.068610:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000040:1.0:1788205023.068614:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205023.068620:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205023.068621:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:1.0:1788205023.068622:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205023.068627:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205023.068628:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205023.068629:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205023.068632:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:1.0:1788205023.068633:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205023.068637:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26709 00000400:00000200:3.0:1788205023.068641:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205023.068642:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205023.068647:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205023.068648:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205023.068651:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000040:1.0:1788205023.068652:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068655:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068658:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205023.068658:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1788205023.068661:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205023.068663:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205023.068666:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000400:00000200:3.0:1788205023.068666:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000001:1.0:1788205023.068668:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205023.068671:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205023.068674:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788205023.068675:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471264:471264:256:4294967295] 192.168.201.118@tcp LPNI seq info [471123:471123:8:4294967295] 00000020:00000001:1.0:1788205023.068676:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205023.068680:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068683:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068687:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205023.068687:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205023.068690:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205023.068692:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205023.068694:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068695:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205023.068697:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205023.068699:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205023.068701:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068705:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205023.068708:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000400:00000200:3.0:1788205023.068709:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000008:00000001:1.0:1788205023.068712:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068716:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205023.068720:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000010:1.0:1788205023.068721:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000020:00000040:1.0:1788205023.068726:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000800:00000010:3.0:1788205023.068728:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 02000000:00000001:1.0:1788205023.068729:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205023.068731:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205023.068737:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205023.068741:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1788205023.068742:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205023.068745:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205023.068748:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788205023.068751:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000010:1.0:1788205023.068754:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb000. 00000800:00000200:3.1:1788205023.068756:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000001:1.0:1788205023.068757:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205023.068765:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788205023.068767:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.068772:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000100:00000001:1.0:1788205023.068774:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205023.068777:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205023.068781:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205023.068781:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1788205023.068791:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205023.068792:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00080000:1.0:1788205023.068794:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205023.068797:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:3.0:1788205023.068798:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000100:00000001:1.0:1788205023.068799:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205023.068802:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788205023.068804:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:1.0:1788205023.068806:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205023.068810:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1788205023.068811:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26709 00000400:00000200:1.0:1788205023.068818:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205023.068822:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000200:3.0:1788205023.068824:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205023.068825:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205023.068831:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068836:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205023.068841:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068844:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205023.068844:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1788205023.068847:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205023.068849:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205023.068851:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:3.0:1788205023.068851:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205023.068852:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205023.068856:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205023.068860:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205023.068866:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205023.068869:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068870:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205023.068871:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1788205023.068875:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205023.068878:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205023.068880:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205023.068881:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205023.068886:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:3.0:1788205023.068888:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205023.068890:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068893:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321383173 02000000:00000001:1.0:1788205023.068894:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205023.068898:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1788205023.068901:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc660. 02000000:00000001:1.0:1788205023.068902:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068906:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205023.068907:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1788205023.068909:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000008:00000001:1.0:1788205023.068910:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068913:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205023.068917:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205023.068918:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321383173 00000020:00000040:1.0:1788205023.068919:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205023.068922:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205023.068924:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.068925:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205023.068927:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:3.0:1788205023.068927:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcf00. 02000000:00000001:1.0:1788205023.068929:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068932:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000010:1.0:1788205023.068934:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8400. 00000100:00000001:3.0:1788205023.068937:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205023.068939:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068941:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.068945:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205023.068946:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788205023.068952:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205023/real 1788205023] req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205023.068962:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205023.068967:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205023.068975:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205023.068979:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205023.068982:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205023.068986:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00000001:1.0:1788205023.068987:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205023.068991:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000100:00000040:1.0:1788205023.068993:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26719) 00000400:00000010:3.0:1788205023.068996:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205023.069000:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205023.069005:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205023.069013:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069021:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205023.069023:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069025:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.069026:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205023.069031:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069037:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205023.069041:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069048:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205023.069051:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069053:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.069054:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069056:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205023.069058:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205023.069061:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205023.069064:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205023.069066:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205023.069070:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205023.069079:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543501184:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205023.069083:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069085:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.069110:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543501184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069118:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205023.069120:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205023.069122:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205023.069126:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205023.069127:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205023.069129:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205023.069131:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205023.069133:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:3.0:1788205023.069137:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205023.069138:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205023.069140:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205023.069146:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205023.069147:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069149:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069156:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205023.069158:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.069160:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205023.069161:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205023.069163:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205023.069166:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069173:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205023.069174:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.069176:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205023.069178:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205023.069180:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205023.069182:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205023.069186:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205023.069187:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205023.069189:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.069191:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205023.069193:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205023.069195:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205023.069199:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205023.069201:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205023.069204:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205023.069207:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543501312, portal 10 00000100:00000001:3.0:1788205023.069209:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205023.069210:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205023.069216:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069222:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205023.069224:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205023.069226:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205023.069228:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543501312, offset 0 00000400:00000200:3.0:1788205023.069232:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205023.069236:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205023.069240:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205023.069242:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205023.069245:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205023.069247:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205023.069249:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205023.069252:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471265:471265:256:4294967295] 192.168.201.118@tcp LPNI seq info [471124:471124:8:4294967295] 00000400:00000200:3.0:1788205023.069256:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205023.069261:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205023.069265:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.069268:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205023.069272:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205023.069275:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205023.069279:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205023.069282:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205023.069285:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.069287:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205023.069290:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205023.069293:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205023.069296:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205023.069298:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205023.069300:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205023.069303:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069310:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.069313:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069319:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.069320:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205023.069324:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205023.069327:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205023.069330:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205023.069332:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321381907 00000100:00000001:3.0:1788205023.069334:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321383173 00000100:00000001:3.0:1788205023.069336:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321381907 00000100:00000001:3.0:1788205023.069338:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321383173 00000100:00000001:3.0:1788205023.069340:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205023.069342:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.069343:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205023.069346:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205023/real 1788205023] req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205023.069354:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205023.069355:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205023.069357:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205023.069359:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205023.069361:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205023.069363:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205023.069366:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205023.069374:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069394:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205023.069395:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069397:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.069398:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205023.069401:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069408:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205023.069412:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069418:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205023.069420:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069422:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.069423:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069424:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205023.069427:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205023.069429:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205023.069432:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205023.069435:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205023.069438:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205023.069446:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205023.069450:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205023.069452:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069458:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205023.069459:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.069461:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205023.069463:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205023.069465:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205023.069468:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205023.069471:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205023.069473:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205023.069475:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.069476:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205023.069480:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205023.069482:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205023.069484:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205023.069487:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205023.069490:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205023.069492:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543501440, portal 10 00000100:00000001:3.0:1788205023.069494:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205023.069496:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205023.069501:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069507:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205023.069509:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205023.069512:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205023.069514:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543501440, offset 0 00000400:00000200:3.0:1788205023.069518:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205023.069522:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205023.069526:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205023.069529:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205023.069531:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205023.069533:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205023.069535:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205023.069538:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471266:471266:256:4294967295] 192.168.201.118@tcp LPNI seq info [471125:471125:8:4294967295] 00000400:00000200:3.0:1788205023.069542:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205023.069547:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205023.069551:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.069555:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205023.069558:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205023.069561:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205023.069566:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205023.069568:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205023.069572:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.069574:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205023.069577:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205023.069580:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205023.069583:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205023.069585:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205023.069587:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205023.069590:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069598:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.069600:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069606:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.069608:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205023.069611:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205023.069614:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205023.069617:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205023.069618:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321381908 00000100:00000001:3.0:1788205023.069621:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321383173 00000100:00000001:3.0:1788205023.069623:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321381908 00000100:00000001:3.0:1788205023.069625:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321383173 00000100:00000001:3.0:1788205023.069627:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205023.069629:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.069631:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205023.069634:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205023/real 1788205023] req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205023.069642:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205023.069643:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205023.069645:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205023.069647:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205023.069648:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205023.069650:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205023.069653:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205023.069661:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069667:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205023.069669:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069670:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.069671:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205023.069675:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069681:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205023.069685:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069691:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205023.069693:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069695:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.069696:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069698:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205023.069701:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205023.069703:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205023.069706:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205023.069708:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205023.069712:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205023.069720:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205023.069723:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205023.069727:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069733:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205023.069734:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.069736:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205023.069738:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205023.069740:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205023.069743:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205023.069746:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205023.069748:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205023.069749:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.069751:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205023.069754:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205023.069756:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205023.069759:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205023.069762:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205023.069765:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205023.069767:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543501568, portal 4 00000100:00000001:3.0:1788205023.069770:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205023.069771:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788205023.069776:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069782:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205023.069785:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205023.069787:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205023.069789:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543501568, offset 0 00000400:00000200:3.0:1788205023.069793:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205023.069797:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205023.069800:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205023.069803:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205023.069805:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205023.069808:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205023.069810:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205023.069813:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471267:471267:256:4294967295] 192.168.201.118@tcp LPNI seq info [471126:471126:8:4294967295] 00000400:00000200:3.0:1788205023.069817:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205023.069823:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205023.069827:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.069830:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205023.069833:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205023.069836:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205023.069839:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205023.069842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205023.069845:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.069847:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205023.069850:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205023.069853:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205023.069855:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205023.069857:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205023.069859:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205023.069862:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069869:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.069872:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069883:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.069884:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205023.069887:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205023.069890:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205023.069893:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205023.069895:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321381917 00000100:00000001:3.0:1788205023.069898:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321383173 00000100:00000001:3.0:1788205023.069900:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321381917 00000100:00000001:3.0:1788205023.069902:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321383173 00000100:00000001:3.0:1788205023.069904:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205023.069906:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.069908:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205023.069911:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205023/real 1788205023] req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205023.069919:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205023.069920:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205023.069922:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205023.069924:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205023.069926:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205023.069928:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205023.069931:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205023.069939:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069945:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205023.069946:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069948:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.069950:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205023.069953:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069960:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205023.069963:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.069970:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205023.069972:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069973:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.069975:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.069976:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205023.069978:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205023.069980:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205023.069983:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205023.069985:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205023.069989:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205023.069996:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205023.069999:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205023.070002:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070008:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205023.070009:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.070011:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205023.070013:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205023.070015:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205023.070017:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205023.070020:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205023.070022:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205023.070024:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205023.070025:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205023.070028:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205023.070030:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205023.070033:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205023.070035:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205023.070038:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205023.070040:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543501696, portal 4 00000100:00000001:3.0:1788205023.070042:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205023.070043:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205023.070048:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070054:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205023.070056:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205023.070059:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205023.070061:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543501696, offset 0 00000400:00000200:3.0:1788205023.070064:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205023.070068:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205023.070071:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205023.070074:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205023.070076:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205023.070078:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205023.070080:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205023.070082:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471268:471268:256:4294967295] 192.168.201.118@tcp LPNI seq info [471127:471127:8:4294967295] 00000400:00000200:3.0:1788205023.070086:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205023.070114:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205023.070119:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.070122:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205023.070125:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205023.070128:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205023.070132:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205023.070134:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205023.070138:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205023.070140:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205023.070143:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205023.070146:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205023.070148:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205023.070150:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205023.070152:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205023.070155:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070162:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.070165:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205048 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070171:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.070173:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205023.070175:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205023.070178:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205023.070181:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205023.070183:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321381941 00000100:00000001:3.0:1788205023.070185:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321383173 00000100:00000001:3.0:1788205023.070187:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321381941 00000100:00000001:3.0:1788205023.070188:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321383173 00000100:00000001:3.0:1788205023.070190:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205023.070192:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.070194:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205023.070197:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205023/real 1788205023] req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205023.070203:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205023.070205:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205023.070206:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205023.070209:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205023.070210:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205023.070212:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205023.070215:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205023.070223:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070228:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205023.070229:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070231:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.070232:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205023.070236:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070241:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205023.070244:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070251:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205023.070253:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070254:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205023.070255:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070257:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205023.070259:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205023.070261:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205023.070264:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205023.070266:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205023.070269:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205023.070276:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205023.070279:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070281:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.070285:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543501312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070291:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205023.070292:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205023.070294:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205023.070296:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205023.070298:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205023.070299:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205023.070301:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205023.070302:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788205023.070305:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205023.070306:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205023.070308:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205023.070311:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205023.070312:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070314:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.070317:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543501440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070324:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205023.070325:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205023.070326:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205023.070328:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205023.070329:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205023.070330:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205023.070332:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205023.070333:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:3.0:1788205023.070335:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205023.070336:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205023.070338:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205023.070340:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205023.070341:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070342:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.070345:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543501568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070351:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205023.070353:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205023.070354:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205023.070356:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205023.070357:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205023.070358:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205023.070360:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205023.070361:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:3.0:1788205023.070363:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205023.070364:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205023.070365:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205023.070369:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205023.070370:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070372:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205023.070375:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543501696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205048 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205023.070393:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205023.070395:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205023.070396:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205023.070399:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205023.070400:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205023.070401:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205023.070403:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205023.070404:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:3.0:1788205023.070405:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205023.070406:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205023.070408:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205023.070411:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205023.070412:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070414:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.070418:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205023.070419:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.070421:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205023.070422:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.900176:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205023.900176:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205023.900179:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205023.900183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.900184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205023.900185:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205023.900188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.900189:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205023.900193:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205023.900195:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00100000:3.0:1788205023.900195:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205023.900198:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205023.900199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205023.900200:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205023.900203:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.900203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205023.900205:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205023.900208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205023.900209:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788205023.900211:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205023.900212:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900215:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205023.900216:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900217:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205023.900219:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205023.900221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205023.900223:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205023.900224:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205023.900226:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205023.900228:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900229:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205023.900235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900237:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900238:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205023.900242:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205023.900243:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205023.900244:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900245:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205023.900246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900247:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205023.900249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205023.900302:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205023.900306:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205023.900309:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205024.412179:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321383173, at 4321383232 00000100:00000040:3.0:1788205024.412179:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321383173, at 4321383232 00000100:00000010:3.0:1788205024.412193:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf00. 00000100:00000010:2.0:1788205024.412195:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc660. 00000100:00000001:3.0:1788205024.924186:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205024.924187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205024.924192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205024.924194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205024.924195:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205024.924197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205024.924197:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205024.924201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205024.924202:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205024.924203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205024.924204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205024.924206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205024.924206:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205024.924207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205024.924211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205024.924211:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205024.924213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205024.924215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205024.924215:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205024.924216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205024.924219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205024.924220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205024.924220:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205024.924222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205024.924225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205024.924225:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205024.924246:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205024.924252:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205024.924255:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205025.948192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205025.948192:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205025.948196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205025.948201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205025.948201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205025.948201:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205025.948205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205025.948205:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205025.948209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205025.948212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205025.948212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205025.948212:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205025.948217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205025.948219:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205025.948220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205025.948221:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205025.948222:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205025.948224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205025.948225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205025.948227:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205025.948228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205025.948230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205025.948232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205025.948232:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205025.948233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205025.948239:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205025.948284:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205025.948286:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205025.948288:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205026.972189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205026.972189:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205026.972195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205026.972197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205026.972197:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205026.972200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205026.972202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.972202:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205026.972205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205026.972208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205026.972210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205026.972210:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205026.972211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205026.972214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205026.972218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205026.972218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205026.972218:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205026.972223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205026.972225:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.972225:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205026.972232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205026.972233:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205026.972236:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205026.972236:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205026.972243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205026.972243:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205026.972287:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205026.972297:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205026.972300:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205027.996184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205027.996184:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205027.996186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205027.996191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205027.996191:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205027.996192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205027.996194:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205027.996195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205027.996197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205027.996200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205027.996201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205027.996202:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205027.996203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205027.996207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205027.996207:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205027.996208:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205027.996210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205027.996212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205027.996213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205027.996213:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205027.996216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205027.996218:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205027.996219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205027.996221:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205027.996225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205027.996225:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205027.996261:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205027.996264:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205027.996266:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205028.188240:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205028.188251:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205028.188261:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205028.188267:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205028.188272:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205028.188279:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26714 00000400:00000200:3.0:1788205028.188289:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205028.188320:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205028.188323:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205028.188326:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188328:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205028.188330:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188331:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205028.188335:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205028.188337:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205028.188339:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205028.188341:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205028.188343:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205028.188345:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205028.188346:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188348:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205028.188349:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188350:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205028.188352:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205028.188354:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188356:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205028.188358:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205028.188361:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205028.188363:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.188365:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205028.188367:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205028.188375:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205028.188378:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205028.188381:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205028.188382:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.188385:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205028.188387:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205028.188391:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205028.188397:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788205028.188401:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205028.188407:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205028.188414:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205028.188417:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205028.188420:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205028.188439:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205028.188441:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205028.188444:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205028.188445:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205028.188446:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205028.188447:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205028.188449:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26714 00000400:00000200:3.0:1788205028.188452:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205028.188454:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205028.188456:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205028.188458:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188460:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205028.188462:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188463:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205028.188464:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205028.188465:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205028.188466:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205028.188467:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1788205028.188468:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205028.188472:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205028.188473:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788205028.188473:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205028.188476:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205028.188477:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188480:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205028.188483:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205028.188484:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205028.188485:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205028.188488:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205028.188492:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188498:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205028.188503:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205028.188504:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205028.188507:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.188510:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205028.188510:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205028.188512:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205028.188517:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205028.188517:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.188523:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1788205028.188524:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543501824:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205028.188528:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1788205028.188530:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788205028.188532:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205028.188535:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788205028.188536:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.188537:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788205028.188540:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205028.188543:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205028.188545:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205028.188546:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788205028.188550:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000400:00000200:1.0:1788205028.188553:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205028.188556:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 02000000:00000001:3.0:1788205028.188560:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205028.188562:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205028.188563:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205028.188567:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205028.188569:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205028.188570:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205028.188572:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543501824, portal 25 00000100:00000001:1.0:1788205028.188575:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000010:3.0:1788205028.188575:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:1.0:1788205028.188577:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000001:3.0:1788205028.188580:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205028.188583:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205028.188588:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205028.188590:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205028.188595:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205028.188599:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205028.188602:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543501824, offset 0 00000400:00000200:1.0:1788205028.188608:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188609:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205028.188613:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.188616:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205028.188621:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205028.188624:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205028.188631:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205028.188636:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205028.188639:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205028.188641:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205028.188645:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471269:471269:256:4294967295] 192.168.201.118@tcp LPNI seq info [471128:471128:8:4294967295] 00000100:00000001:3.0:1788205028.188645:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.188652:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205028.188653:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205028.188658:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205028.188662:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1788205028.188663:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:3.0:1788205028.188666:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1788205028.188671:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188672:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:1.0:1788205028.188676:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000100:00080000:3.0:1788205028.188677:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26714 00000800:00000200:1.0:1788205028.188683:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:3.0:1788205028.188686:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205028.188691:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:1.0:1788205028.188692:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00080000:3.0:1788205028.188693:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205028.188698:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.1:1788205028.188701:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205028.188702:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1788205028.188705:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205028.188709:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.188711:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000100:00000040:3.0:1788205028.188713:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188717:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205028.188718:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1788205028.188722:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000200:1.0:1788205028.188726:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000020:00000040:3.0:1788205028.188726:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:1.0:1788205028.188730:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000020:00000001:3.0:1788205028.188730:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205028.188733:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000010:1.0:1788205028.188734:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000020:00000001:3.0:1788205028.188738:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788205028.188740:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1788205028.188742:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000200:1.0:1788205028.188745:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205028.188751:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205028.188755:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205028.188755:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205028.188760:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205028.188761:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205028.188766:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205028.188770:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.188772:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:3.0:1788205028.188772:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000001:1.0:1788205028.188778:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.188786:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205028.188786:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205028.188788:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321388293 00000100:00000001:3.0:1788205028.188792:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205028.188795:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647c60. 02000000:00000001:3.0:1788205028.188796:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205028.188800:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205028.188802:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.188805:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205028.188807:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205028.188809:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.188811:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205028.188811:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.188818:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321388293 00000100:00000010:3.0:1788205028.188820:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000100:00000010:1.0:1788205028.188822:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647420. 00000020:00000040:3.0:1788205028.188824:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000100:00000001:1.0:1788205028.188826:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:3.0:1788205028.188830:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205028.188831:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205028.188834:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205028.188834:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.188839:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788205028.188839:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205028.188842:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788205028.188843:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205028/real 1788205028] req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205028.188845:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205028.188852:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000400:00000001:1.0:1788205028.188854:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205028.188856:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788205028.188857:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205028.188858:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205028.188862:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205028.188863:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205028.188864:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205028.188867:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205028.188876:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205028.188880:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205028.188896:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205028.188899:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205028.188900:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.188903:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.188908:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1788205028.188908:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.188912:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205028.188920:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205028.188923:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.188929:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205028.188930:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1788205028.188934:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205028.188935:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205028.188938:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205028.188940:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205028.188945:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205028.188946:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205028.188950:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205028.188951:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26714 00000100:00000001:1.0:1788205028.188952:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.188956:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1788205028.188959:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205028.188960:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205028.188964:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:3.0:1788205028.188966:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205028.188967:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205028.188969:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788205028.188973:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000040:3.0:1788205028.188976:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205028.188977:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788205028.188981:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205028.188985:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205028.188986:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.188990:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205028.188994:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205028.188997:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205028.188999:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:1.0:1788205028.189001:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543501824:192.168.201.118@tcp:250:kworker.0 00000020:00000040:3.0:1788205028.189002:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205028.189007:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205028.189009:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788205028.189009:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205028.189012:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205028.189015:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205028.189019:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543501824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205028.189019:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205028.189024:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.189027:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205028.189031:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205028.189034:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205028.189035:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205028.189036:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205028.189038:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205028.189039:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.189040:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205028.189041:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000008:00100000:3.0:1788205028.189042:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205028.189045:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.189046:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788205028.189047:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205028.189051:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 02000000:00000010:1.0:1788205028.189052:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 00000020:00000040:3.0:1788205028.189055:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205028.189058:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1788205028.189058:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205028.189061:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205028.189063:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000001:3.0:1788205028.189063:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000020:00000040:1.0:1788205028.189067:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205028.189067:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205028.189069:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205028.189074:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205028.189075:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 02000000:00000010:1.0:1788205028.189079:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788205028.189079:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205028.189085:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205028.189086:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205028.189115:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205028.189117:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:3.0:1788205028.189125:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205028.189127:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205028.189127:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.189131:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205028.189133:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189136:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205028.189142:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205028.189144:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205028.189147:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205028.189148:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205028.189149:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205028.189150:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205028.189152:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1788205028.189153:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26714 00000100:00000001:1.0:1788205028.189156:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205028.189160:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205028.189162:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1788205028.189163:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205028.189165:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788205028.189166:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205028.189169:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205028.189170:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.189174:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205028.189178:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205028.189179:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205028.189184:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205028.189189:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:1.0:1788205028.189191:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205028.189193:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205028.189193:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 02000000:00000001:1.0:1788205028.189196:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1788205028.189197:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:1.0:1788205028.189198:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205028.189201:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205028.189202:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00100000:1.0:1788205028.189205:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501952:192.168.201.118@tcp:38:kworker.0 00000020:00000001:3.0:1788205028.189207:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788205028.189209:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205028.189211:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1788205028.189212:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 02000000:00000001:1.0:1788205028.189213:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.189216:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205028.189220:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000040:3.0:1788205028.189221:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205028.189222:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.189227:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205028.189230:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205028.189232:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:1.0:1788205028.189234:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000100:00000001:3.0:1788205028.189235:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000010:1.0:1788205028.189239:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000020:00000040:3.0:1788205028.189239:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000200:1.0:1788205028.189245:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543501952, portal 10 00000100:00080000:3.0:1788205028.189245:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205028.189248:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.189249:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788205028.189251:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205028.189252:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 02000000:00000001:3.0:1788205028.189257:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205028.189260:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205028.189262:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.189268:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205028.189271:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:3.0:1788205028.189271:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000040:1.0:1788205028.189275:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205028.189278:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000008:00000001:3.0:1788205028.189278:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205028.189283:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543501952, offset 0 00000100:00000001:3.0:1788205028.189284:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.189288:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000010:3.0:1788205028.189303:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000400:00000200:1.0:1788205028.189307:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:3.0:1788205028.189307:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205028.189309:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205028.189311:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205028.189313:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205028.189315:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1788205028.189316:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 02000000:00000001:3.0:1788205028.189317:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.189322:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:3.0:1788205028.189323:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000400:00000200:1.0:1788205028.189327:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:3.0:1788205028.189327:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.189333:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00100000:3.0:1788205028.189335:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205028.189339:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205028.189348:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471270:471270:256:4294967295] 192.168.201.118@tcp LPNI seq info [471129:471129:8:4294967295] 00000100:00000001:3.0:1788205028.189348:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205028.189352:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.189356:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205028.189359:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205028.189372:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788205028.189379:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205028.189381:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.189385:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000100:00000040:3.0:1788205028.189386:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26724) 00000800:00000200:1.0:1788205028.189390:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205028.189393:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205028.189396:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205028.189398:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205028.189402:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.189404:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205028.189406:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205028.189409:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205028.189413:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205028.189415:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205028.189418:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205028.189423:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189437:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205028.189440:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189446:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.189447:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205028.189451:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205028.189454:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205028.189457:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205028.189459:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321387027 00000100:00000001:1.0:1788205028.189462:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321388293 00000100:00000001:1.0:1788205028.189464:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321387027 00000100:00000001:1.0:1788205028.189466:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321388293 00000100:00000001:1.0:1788205028.189467:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205028.189470:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.189472:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205028.189475:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205028/real 1788205028] req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205028.189483:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205028.189484:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205028.189486:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205028.189488:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205028.189490:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205028.189492:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205028.189495:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205028.189503:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189509:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205028.189512:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.189513:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.189515:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205028.189518:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189524:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205028.189527:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189531:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205028.189533:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.189534:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.189534:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.189535:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205028.189537:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205028.189538:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205028.189540:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205028.189541:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205028.189544:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205028.189551:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543501952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205028.189555:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205028.189558:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189564:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205028.189565:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.189567:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205028.189569:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205028.189570:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205028.189573:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205028.189576:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205028.189577:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205028.189579:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.189580:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205028.189584:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205028.189586:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.189589:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205028.189591:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205028.189594:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205028.189596:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543502080, portal 10 00000100:00000001:1.0:1788205028.189598:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205028.189599:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205028.189604:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189610:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205028.189612:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205028.189614:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205028.189616:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543502080, offset 0 00000400:00000200:1.0:1788205028.189620:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205028.189624:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205028.189627:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205028.189630:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205028.189633:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205028.189635:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205028.189637:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205028.189640:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471271:471271:256:4294967295] 192.168.201.118@tcp LPNI seq info [471130:471130:8:4294967295] 00000400:00000200:1.0:1788205028.189644:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205028.189649:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205028.189653:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.189656:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205028.189660:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205028.189663:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205028.189667:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205028.189669:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205028.189673:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.189675:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205028.189678:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205028.189681:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205028.189684:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205028.189685:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205028.189687:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205028.189690:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189697:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205028.189700:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189705:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.189707:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205028.189709:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205028.189712:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205028.189715:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205028.189717:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321387028 00000100:00000001:1.0:1788205028.189719:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321388293 00000100:00000001:1.0:1788205028.189721:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321387028 00000100:00000001:1.0:1788205028.189723:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321388293 00000100:00000001:1.0:1788205028.189725:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205028.189727:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.189729:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205028.189733:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205028/real 1788205028] req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205028.189740:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205028.189742:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205028.189743:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205028.189746:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205028.189747:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205028.189749:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205028.189752:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205028.189759:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189765:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205028.189766:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.189768:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.189769:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205028.189772:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189778:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205028.189782:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189788:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205028.189791:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.189792:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.189794:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.189795:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205028.189797:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205028.189800:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205028.189803:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205028.189805:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205028.189808:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205028.189820:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205028.189823:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205028.189825:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189829:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205028.189830:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.189831:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205028.189833:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205028.189834:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205028.189835:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205028.189837:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205028.189839:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205028.189840:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.189841:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205028.189843:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205028.189844:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.189847:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205028.189848:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205028.189850:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205028.189852:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543502208, portal 4 00000100:00000001:1.0:1788205028.189854:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205028.189854:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205028.189858:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189863:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205028.189865:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205028.189867:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205028.189869:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543502208, offset 0 00000400:00000200:1.0:1788205028.189873:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205028.189876:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205028.189879:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205028.189882:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205028.189884:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205028.189886:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205028.189888:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205028.189891:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471272:471272:256:4294967295] 192.168.201.118@tcp LPNI seq info [471131:471131:8:4294967295] 00000400:00000200:1.0:1788205028.189895:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205028.189901:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205028.189906:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.189909:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205028.189912:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205028.189915:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205028.189919:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205028.189920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205028.189923:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.189924:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205028.189926:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205028.189928:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205028.189930:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205028.189931:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205028.189932:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205028.189935:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189940:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205028.189941:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189945:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.189946:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205028.189948:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205028.189950:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205028.189952:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205028.189953:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321387037 00000100:00000001:1.0:1788205028.189955:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321388293 00000100:00000001:1.0:1788205028.189956:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321387037 00000100:00000001:1.0:1788205028.189958:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321388293 00000100:00000001:1.0:1788205028.189959:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205028.189960:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.189961:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205028.189963:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205028/real 1788205028] req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205028.189967:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205028.189968:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205028.189969:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205028.189970:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205028.189972:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205028.189973:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205028.189974:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205028.189979:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189982:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205028.189983:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.189984:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.189985:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205028.189986:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189990:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205028.189992:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.189996:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205028.189997:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.189998:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.189999:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190000:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205028.190001:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205028.190003:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205028.190004:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205028.190006:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205028.190008:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205028.190011:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205028.190014:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190015:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205028.190016:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543501952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190020:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205028.190021:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205028.190022:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205028.190024:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205028.190025:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205028.190025:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205028.190027:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205028.190028:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205028.190029:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205028.190030:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205028.190031:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205028.190033:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205028.190034:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190035:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205028.190037:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543502080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190047:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205028.190048:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205028.190048:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205028.190050:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205028.190051:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205028.190052:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205028.190052:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205028.190053:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205028.190054:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205028.190055:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205028.190056:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205028.190057:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205028.190058:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190059:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205028.190060:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543502208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190064:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205028.190064:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205028.190065:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205028.190066:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205028.190067:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205028.190067:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205028.190068:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205028.190069:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788205028.190070:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205028.190071:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205028.190071:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205028.190072:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205028.190073:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190074:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190076:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205028.190077:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.190078:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205028.190079:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205028.190080:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205028.190082:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190086:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205028.190087:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.190103:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205028.190104:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205028.190106:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205028.190107:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205028.190110:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205028.190111:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205028.190112:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205028.190113:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205028.190115:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205028.190116:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205028.190118:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205028.190120:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205028.190122:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205028.190124:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543502336, portal 4 00000100:00000001:1.0:1788205028.190125:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205028.190126:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205028.190130:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190134:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205028.190135:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205028.190136:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205028.190138:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543502336, offset 0 00000400:00000200:1.0:1788205028.190140:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205028.190142:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205028.190144:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205028.190146:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205028.190148:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205028.190149:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205028.190150:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205028.190153:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471273:471273:256:4294967295] 192.168.201.118@tcp LPNI seq info [471132:471132:8:4294967295] 00000400:00000200:1.0:1788205028.190155:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205028.190159:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205028.190162:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.190164:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205028.190166:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205028.190168:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205028.190170:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205028.190172:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205028.190174:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205028.190175:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205028.190177:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205028.190179:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205028.190181:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205028.190182:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205028.190183:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205028.190185:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190190:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205028.190191:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205053 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190195:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.190196:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205028.190198:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205028.190200:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205028.190201:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205028.190203:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321387061 00000100:00000001:1.0:1788205028.190204:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321388293 00000100:00000001:1.0:1788205028.190206:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321387061 00000100:00000001:1.0:1788205028.190207:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321388293 00000100:00000001:1.0:1788205028.190208:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205028.190210:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.190211:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205028.190213:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205028/real 1788205028] req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205028.190217:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205028.190218:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205028.190220:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205028.190221:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205028.190222:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205028.190223:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205028.190224:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205028.190229:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190232:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205028.190233:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190234:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.190235:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205028.190237:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190240:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205028.190242:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190245:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205028.190247:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190247:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205028.190248:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190249:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205028.190250:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205028.190251:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205028.190253:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205028.190255:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205028.190257:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205028.190261:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205028.190263:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190264:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205028.190265:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543502336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205053 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205028.190268:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205028.190269:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205028.190270:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205028.190271:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205028.190272:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205028.190273:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205028.190274:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205028.190274:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205028.190276:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205028.190276:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205028.190277:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205028.190278:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205028.190279:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190280:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205028.190282:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205028.190282:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205028.190284:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205028.190285:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205029.020167:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205029.020170:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205029.020174:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205029.020176:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205029.020176:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205029.020180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205029.020181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205029.020182:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205029.020184:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205029.020186:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205029.020187:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205029.020189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205029.020190:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205029.020191:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205029.020193:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205029.020195:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205029.020196:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205029.020197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205029.020199:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788205029.020200:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205029.020201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205029.020202:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205029.020204:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205029.020207:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205029.020208:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205029.020213:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205029.020216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205029.020218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205029.020219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205029.020220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205029.020223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205029.020224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205029.020275:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205029.020303:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205029.020309:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205029.532226:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321388293, at 4321388352 00000100:00000040:2.0:1788205029.532228:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321388293, at 4321388352 00000100:00000010:3.0:1788205029.532245:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647420. 00000100:00000010:2.0:1788205029.532249:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00000100:00000001:1.0:1788205030.044239:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205030.044239:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205030.044246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205030.044247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205030.044249:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205030.044250:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205030.044252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205030.044255:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205030.044257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205030.044277:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205030.044278:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205030.044281:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205030.044283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205030.044286:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205030.044286:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205030.044288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205030.044289:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205030.044292:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205030.044296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205030.044296:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205030.044297:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205030.044303:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205030.044304:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205030.044314:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205030.044318:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205030.044322:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205030.044383:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205030.044387:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205030.044390:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205031.068197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205031.068197:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205031.068209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205031.068209:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205031.068215:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205031.068215:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205031.068224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205031.068224:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205031.068229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205031.068232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205031.068232:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205031.068239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205031.068239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205031.068240:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205031.068258:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205031.068259:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205031.068265:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205031.068265:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205031.068267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205031.068271:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205031.068272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205031.068272:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205031.068286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205031.068289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205031.068293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205031.068295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205031.068355:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205031.068359:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205031.068363:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205032.092262:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205032.092262:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205032.092264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205032.092274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205032.092276:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205032.092276:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205032.092281:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205032.092281:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205032.092285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205032.092290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205032.092291:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205032.092291:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205032.092303:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205032.092305:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205032.092307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205032.092309:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205032.092309:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205032.092312:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205032.092319:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205032.092319:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205032.092321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205032.092326:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205032.092328:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205032.092329:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205032.092332:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205032.092341:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205032.092422:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205032.092427:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205032.092431:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205033.116194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205033.116194:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205033.116204:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205033.116222:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205033.116224:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.116226:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205033.116231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205033.116232:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205033.116239:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205033.116239:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205033.116245:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.116246:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205033.116250:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205033.116251:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205033.116256:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205033.116257:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.116264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.116264:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205033.117216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205033.117222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205033.117227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205033.117229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205033.117233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205033.117234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205033.117238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205033.117239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205033.117298:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205033.117303:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205033.117307:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205033.308311:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205033.308324:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205033.308327:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205033.308332:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205033.308337:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205033.308354:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26719 00000400:00000200:3.0:1788205033.308371:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205033.308379:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205033.308383:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205033.308386:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308389:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.308391:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308394:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205033.308399:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205033.308401:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205033.308404:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205033.308408:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205033.308410:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205033.308413:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205033.308417:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308419:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.308421:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308423:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205033.308426:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205033.308430:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308432:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205033.308435:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205033.308439:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205033.308442:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308445:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205033.308448:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205033.308461:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205033.308464:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205033.308468:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205033.308470:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308474:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205033.308478:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205033.308482:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205033.308491:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412400. 00000100:00000001:3.0:1788205033.308499:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205033.308508:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205033.308519:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205033.308523:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205033.308528:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205033.308552:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205033.308556:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205033.308560:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205033.308561:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205033.308563:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205033.308566:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205033.308569:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26719 00000400:00000200:3.0:1788205033.308572:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205033.308575:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205033.308578:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205033.308581:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308583:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.308585:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308588:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205033.308590:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205033.308592:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205033.308594:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205033.308597:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205033.308599:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205033.308601:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788205033.308602:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205033.308608:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1788205033.308608:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205033.308613:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788205033.308614:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.308618:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308622:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788205033.308624:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205033.308626:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205033.308633:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308638:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.308641:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788205033.308642:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205033.308645:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205033.308650:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205033.308651:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205033.308655:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205033.308656:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.308660:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308660:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1788205033.308667:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543502464:192.168.201.118@tcp:250:kworker.0 00000008:00100000:3.0:1788205033.308667:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788205033.308674:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205033.308678:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205033.308681:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.308686:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1788205033.308686:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308691:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205033.308695:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 00000100:00000010:3.0:1788205033.308697:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788205033.308701:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205033.308703:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205033.308704:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.308709:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308709:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205033.308711:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205033.308713:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205033.308719:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413800. 00000400:00000200:1.0:1788205033.308723:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205033.308724:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205033.308729:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000100:00100000:3.0:1788205033.308732:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1788205033.308740:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205033.308741:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205033.308745:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205033.308749:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205033.308751:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543502464, portal 25 00000100:00000001:1.0:1788205033.308757:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205033.308760:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000001:3.0:1788205033.308763:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205033.308769:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205033.308769:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205033.308777:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205033.308782:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205033.308784:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205033.308787:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788205033.308789:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1788205033.308794:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205033.308797:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26719 00000400:00000010:1.0:1788205033.308800:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000400:00000200:3.0:1788205033.308805:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:1.0:1788205033.308807:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543502464, offset 0 00000400:00000200:3.0:1788205033.308810:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205033.308814:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205033.308818:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205033.308821:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308822:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.308826:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308829:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205033.308831:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205033.308832:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205033.308834:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205033.308836:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205033.308838:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205033.308840:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205033.308843:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308845:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.308847:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308850:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205033.308852:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205033.308855:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308857:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205033.308859:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205033.308862:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205033.308864:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308866:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205033.308868:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205033.308870:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308872:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205033.308875:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205033.308877:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205033.308880:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205033.308881:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308883:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205033.308884:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205033.308886:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205033.308890:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:3.0:1788205033.308893:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205033.308898:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205033.308905:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205033.308907:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205033.308911:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205033.308919:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205033.308922:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205033.308926:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205033.308927:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205033.308928:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205033.308932:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205033.308937:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205033.308937:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26719 00000400:00000200:3.0:1788205033.308943:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205033.308945:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205033.308947:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205033.308950:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308952:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.308953:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308955:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205033.308958:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205033.308959:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205033.308961:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205033.308963:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205033.308964:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205033.308966:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205033.308969:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308971:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.308972:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308974:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205033.308976:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205033.308979:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205033.308981:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205033.308984:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205033.308986:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205033.308988:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.308992:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205033.308995:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205033.308998:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205033.309000:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788205033.309001:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205033.309008:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000400:00000200:1.0:1788205033.309010:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1788205033.309012:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205033.309016:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1788205033.309017:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000001:3.0:1788205033.309018:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.309022:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1788205033.309024:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1788205033.309025:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205033.309029:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205033.309035:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788205033.309039:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205033.309043:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205033.309051:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205033.309052:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205033.309056:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205033.309063:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205033.309066:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205033.309069:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205033.309070:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205033.309072:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205033.309075:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205033.309077:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26719 00000400:00000200:3.0:1788205033.309080:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205033.309082:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:1.0:1788205033.309086:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00080000:3.0:1788205033.309086:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205033.309116:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.309118:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.309120:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.309123:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205033.309125:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205033.309126:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205033.309128:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205033.309130:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205033.309133:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000200:1.0:1788205033.309137:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471274:471274:256:4294967295] 192.168.201.118@tcp LPNI seq info [471133:471133:8:4294967295] 00000020:00000001:3.0:1788205033.309137:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205033.309145:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205033.309149:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205033.309150:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205033.309154:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205033.309157:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205033.309160:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205033.309167:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:1.0:1788205033.309168:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788205033.309172:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205033.309176:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205033.309179:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205033.309181:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.309183:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205033.309185:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205033.309187:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.309188:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205033.309192:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205033.309194:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205033.309197:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205033.309198:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205033.309199:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205033.309201:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205033.309203:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205033.309218:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:3.0:1788205033.309222:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205033.309226:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205033.309233:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205033.309235:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205033.309238:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205033.309246:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205033.309251:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26729) 00000800:00000200:1.0:1788205033.309525:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.309532:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205033.309541:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205033.309544:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205033.309551:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205033.309555:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205033.309559:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.309562:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205033.309569:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205033.309574:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205033.309581:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205033.309584:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205033.309589:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205033.309594:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.309605:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.309608:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.309616:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.309618:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205033.309627:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205033.309632:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205033.309635:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321393414 00000100:00000010:1.0:1788205033.309640:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647000. 00000100:00000001:1.0:1788205033.309647:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205033.309704:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321393414 00000100:00000010:1.0:1788205033.309709:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647180. 00000100:00000001:1.0:1788205033.309712:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205033.309716:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205033.309718:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.309721:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205033.309726:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205033/real 1788205033] req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205033.309737:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205033.309739:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205033.309742:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205033.309744:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205033.309746:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205033.309748:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205033.309752:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205033.309760:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.309767:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205033.309769:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.309771:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.309773:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205033.309777:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.309784:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205033.309788:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.309795:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205033.309798:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.309800:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.309801:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.309804:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205033.309806:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205033.309810:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205033.309814:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205033.309816:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205033.309820:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205033.309842:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543502464:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205033.309848:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.309850:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.309853:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543502464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.309860:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205033.309863:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205033.309865:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205033.309869:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205033.309871:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205033.309873:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205033.309875:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205033.309877:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:1.0:1788205033.309882:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205033.309883:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205033.309885:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205033.309889:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205033.309890:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.309893:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.309901:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205033.309903:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.309905:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205033.309907:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205033.309908:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205033.309912:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.309919:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205033.309921:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.309923:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205033.309925:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205033.309927:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205033.309930:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502592:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205033.309934:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205033.309936:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205033.309938:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.309940:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205033.309953:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205033.309955:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205033.309959:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205033.309962:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205033.309966:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205033.309969:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543502592, portal 10 00000100:00000001:1.0:1788205033.309971:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205033.309973:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205033.309979:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.309986:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205033.309988:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205033.309990:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205033.309993:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543502592, offset 0 00000400:00000200:1.0:1788205033.309997:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205033.310001:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205033.310006:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205033.310009:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205033.310011:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205033.310014:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205033.310016:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205033.310019:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471275:471275:256:4294967295] 192.168.201.118@tcp LPNI seq info [471134:471134:8:4294967295] 00000400:00000200:1.0:1788205033.310023:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205033.310029:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205033.310043:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.310046:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205033.310050:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205033.310052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205033.310056:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205033.310058:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205033.310062:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.310064:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205033.310067:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205033.310070:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205033.310073:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205033.310074:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205033.310076:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205033.310079:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310086:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.310088:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310110:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.310112:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205033.310116:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205033.310120:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205033.310123:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205033.310125:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321392148 00000100:00000001:1.0:1788205033.310127:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321393414 00000100:00000001:1.0:1788205033.310129:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321392148 00000100:00000001:1.0:1788205033.310131:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321393414 00000100:00000001:1.0:1788205033.310134:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205033.310136:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.310138:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205033.310141:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205033/real 1788205033] req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205033.310148:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205033.310149:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205033.310151:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205033.310153:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205033.310155:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205033.310156:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205033.310159:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205033.310166:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310172:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205033.310173:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310175:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.310176:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205033.310184:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310189:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205033.310192:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310199:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205033.310201:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310202:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.310213:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310215:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205033.310218:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205033.310220:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205033.310223:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205033.310225:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205033.310229:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205033.310235:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502592:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205033.310239:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205033.310242:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310247:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205033.310249:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.310251:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205033.310252:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205033.310254:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205033.310256:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502720:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205033.310259:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205033.310261:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205033.310263:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.310264:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205033.310268:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205033.310269:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205033.310272:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205033.310274:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205033.310277:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205033.310279:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543502720, portal 10 00000100:00000001:1.0:1788205033.310281:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205033.310282:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205033.310288:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310293:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205033.310296:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205033.310298:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205033.310300:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543502720, offset 0 00000400:00000200:1.0:1788205033.310303:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205033.310307:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205033.310310:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205033.310313:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205033.310315:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205033.310317:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205033.310319:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205033.310322:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471276:471276:256:4294967295] 192.168.201.118@tcp LPNI seq info [471135:471135:8:4294967295] 00000400:00000200:1.0:1788205033.310325:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205033.310331:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205033.310335:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.310337:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205033.310341:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205033.310343:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205033.310346:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205033.310349:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205033.310352:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.310354:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205033.310356:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205033.310359:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205033.310362:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205033.310363:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205033.310365:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205033.310368:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310374:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.310377:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310383:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.310385:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205033.310387:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205033.310390:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205033.310392:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205033.310394:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321392149 00000100:00000001:1.0:1788205033.310396:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321393414 00000100:00000001:1.0:1788205033.310398:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321392149 00000100:00000001:1.0:1788205033.310400:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321393414 00000100:00000001:1.0:1788205033.310402:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205033.310404:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.310405:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205033.310408:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205033/real 1788205033] req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205033.310414:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205033.310415:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205033.310416:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205033.310419:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205033.310420:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205033.310422:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205033.310424:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205033.310431:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310437:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205033.310438:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310439:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.310440:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205033.310443:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310449:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205033.310452:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310458:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205033.310460:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310461:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.310462:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310463:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205033.310466:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205033.310467:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205033.310470:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205033.310472:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205033.310475:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205033.310481:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502720:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205033.310485:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205033.310487:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310493:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205033.310494:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.310496:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205033.310498:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205033.310499:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205033.310502:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502848:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205033.310505:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205033.310506:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205033.310508:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.310510:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205033.310512:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205033.310514:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205033.310518:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205033.310520:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205033.310523:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205033.310525:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543502848, portal 4 00000100:00000001:1.0:1788205033.310527:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205033.310528:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788205033.310532:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310538:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205033.310540:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205033.310542:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205033.310543:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543502848, offset 0 00000400:00000200:1.0:1788205033.310546:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205033.310550:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205033.310552:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205033.310555:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205033.310557:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205033.310559:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205033.310561:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205033.310563:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471277:471277:256:4294967295] 192.168.201.118@tcp LPNI seq info [471136:471136:8:4294967295] 00000400:00000200:1.0:1788205033.310566:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205033.310572:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205033.310576:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.310578:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205033.310581:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205033.310583:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205033.310587:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205033.310589:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205033.310592:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.310594:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205033.310596:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205033.310599:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205033.310601:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205033.310603:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205033.310606:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205033.310609:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310616:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.310619:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310625:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.310627:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205033.310629:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205033.310632:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205033.310635:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205033.310636:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321392158 00000100:00000001:1.0:1788205033.310638:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321393414 00000100:00000001:1.0:1788205033.310641:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321392158 00000100:00000001:1.0:1788205033.310643:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321393414 00000100:00000001:1.0:1788205033.310645:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205033.310646:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.310648:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205033.310651:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205033/real 1788205033] req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205033.310657:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205033.310658:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205033.310659:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205033.310661:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205033.310662:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205033.310664:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205033.310667:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205033.310674:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310681:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205033.310682:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310684:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.310685:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205033.310688:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310695:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205033.310698:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310705:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205033.310707:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310709:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.310710:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310712:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205033.310714:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205033.310716:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205033.310720:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205033.310723:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205033.310726:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205033.310733:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502848:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205033.310737:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205033.310740:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310745:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205033.310747:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.310749:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205033.310751:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205033.310753:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205033.310755:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502976:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205033.310758:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205033.310760:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205033.310761:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205033.310763:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205033.310766:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205033.310768:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205033.310771:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205033.310774:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000400:00000010:1.0:1788205033.310777:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205033.310780:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543502976, portal 4 00000100:00000001:1.0:1788205033.310782:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205033.310783:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788205033.310788:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310794:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205033.310796:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205033.310798:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000200:1.0:1788205033.310801:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543502976, offset 0 00000400:00000200:1.0:1788205033.310804:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205033.310808:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205033.310811:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205033.310813:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205033.310815:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205033.310817:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205033.310819:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205033.310822:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471278:471278:256:4294967295] 192.168.201.118@tcp LPNI seq info [471137:471137:8:4294967295] 00000400:00000200:1.0:1788205033.310826:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205033.310831:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205033.310837:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.310839:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000800:00000200:1.0:1788205033.310842:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205033.310845:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205033.310849:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205033.310851:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205033.310855:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205033.310857:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:1.0:1788205033.310860:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205033.310863:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205033.310865:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:1.0:1788205033.310867:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:1.0:1788205033.310869:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205033.310872:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310879:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.310882:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205058 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310888:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.310889:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205033.310892:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205033.310895:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205033.310897:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205033.310900:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321392181 00000100:00000001:1.0:1788205033.310902:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321393414 00000100:00000001:1.0:1788205033.310904:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321392181 00000100:00000001:1.0:1788205033.310906:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321393414 00000100:00000001:1.0:1788205033.310908:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205033.310910:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.310911:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205033.310914:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205033/real 1788205033] req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205033.310922:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205033.310924:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205033.310925:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205033.310927:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205033.310929:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205033.310931:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205033.310933:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205033.310940:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310946:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205033.310948:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310949:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.310950:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205033.310954:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310960:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205033.310963:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.310969:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205033.310971:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310972:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205033.310974:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.310975:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205033.310977:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205033.310979:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205033.310982:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205033.310984:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205033.310988:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205033.310995:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543502976:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205033.310999:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.311000:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.311003:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543502592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.311009:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205033.311011:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205033.311013:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205033.311015:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205033.311017:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205033.311018:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205033.311021:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205033.311023:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:1.0:1788205033.311025:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205033.311026:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205033.311028:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205033.311030:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205033.311031:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.311033:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.311036:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543502720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.311041:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205033.311042:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205033.311044:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205033.311046:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205033.311048:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205033.311049:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205033.311051:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205033.311052:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205033.311054:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205033.311056:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205033.311057:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205033.311059:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205033.311060:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.311062:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.311065:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543502848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.311070:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205033.311071:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205033.311073:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205033.311075:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205033.311076:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205033.311077:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205033.311079:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205033.311080:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205033.311082:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205033.311083:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205033.311085:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205033.311087:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205033.311088:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.311108:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205033.311111:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543502976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205058 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205033.311117:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205033.311119:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205033.311120:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205033.311122:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205033.311124:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205033.311125:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205033.311127:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205033.311128:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788205033.311130:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205033.311131:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205033.311132:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205033.311136:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205033.311137:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.311139:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205033.311144:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205033.311145:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205033.311147:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205033.311149:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205034.140237:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205034.140237:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205034.140242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205034.140247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205034.140248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205034.140248:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205034.140250:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205034.140252:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205034.140254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205034.140256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205034.140257:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205034.140260:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205034.140261:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205034.140262:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205034.140264:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205034.140265:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205034.140267:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205034.140268:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205034.140269:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205034.140270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205034.140275:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205034.140276:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788205034.140276:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140279:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205034.140280:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205034.140283:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205034.140285:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140286:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205034.140294:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205034.140295:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205034.140296:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140297:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205034.140298:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140299:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140300:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205034.140304:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205034.140306:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205034.140307:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140308:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205034.140309:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140310:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205034.140311:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205034.140369:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205034.140374:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205034.140377:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205034.652260:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321393414, at 4321393472 00000100:00000010:2.0:1788205034.652281:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647000. 00000100:00000040:3.0:1788205034.652514:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321393414, at 4321393472 00000100:00000010:3.0:1788205034.652521:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00000100:00000001:1.0:1788205035.164252:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205035.164259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205035.164263:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205035.164265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205035.164267:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205035.164277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205035.164279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205035.164282:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205035.164284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205035.164287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205035.164288:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205035.164293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205035.164294:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205035.164298:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205035.164299:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205035.164306:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205035.164310:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205035.164373:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205035.164378:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205035.164379:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205035.164382:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205035.164385:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205035.164387:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205035.164391:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205035.164394:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205035.164396:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205035.164412:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205035.164417:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205035.164421:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205036.188247:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205036.188247:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205036.188253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205036.188258:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205036.188259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205036.188259:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205036.188263:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.188263:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205036.188269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205036.188271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205036.188271:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205036.188272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205036.188279:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205036.188281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205036.188283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205036.188284:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205036.188287:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205036.188289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205036.188290:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205036.188291:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205036.188292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205036.188294:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205036.188298:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205036.188300:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205036.188303:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205036.188307:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205036.188366:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205036.188370:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205036.188374:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205037.212212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205037.212212:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205037.212220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205037.212224:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205037.212224:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205037.212227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205037.212229:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205037.212229:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205037.212236:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205037.212240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205037.212242:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205037.212243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205037.212243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205037.212247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205037.212249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205037.212250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205037.212252:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205037.212254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205037.212254:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205037.212258:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205037.212261:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205037.212263:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205037.212264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205037.212266:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205037.212268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205037.212271:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205037.212346:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205037.212351:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205037.212355:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205038.236234:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205038.236241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205038.236244:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205038.236247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.236248:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205038.236257:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205038.236259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205038.236261:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205038.236264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.236266:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205038.236269:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205038.236270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205038.236272:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205038.236275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.236276:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205038.236284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205038.236286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205038.236368:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205038.236374:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205038.236378:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205038.236379:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205038.236384:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205038.236385:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205038.236387:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205038.236389:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205038.236391:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.236393:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205038.236395:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.236397:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205038.428270:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205038.428283:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205038.428287:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205038.428291:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205038.428296:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205038.428304:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26724 00000400:00000200:3.0:1788205038.428316:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205038.428325:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205038.428328:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205038.428331:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428333:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205038.428335:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428337:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205038.428341:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205038.428343:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205038.428345:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205038.428348:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205038.428350:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205038.428351:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205038.428353:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428354:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205038.428355:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428357:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205038.428359:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205038.428362:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428363:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205038.428366:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205038.428369:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205038.428371:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428372:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205038.428375:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205038.428385:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205038.428388:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205038.428391:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205038.428392:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428395:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205038.428398:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205038.428403:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205038.428412:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:3.0:1788205038.428428:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205038.428443:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205038.428455:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205038.428460:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205038.428465:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205038.428494:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205038.428497:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205038.428500:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205038.428502:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205038.428504:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205038.428506:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205038.428509:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26724 00000400:00000200:3.0:1788205038.428514:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205038.428517:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205038.428520:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205038.428524:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428526:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205038.428529:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428531:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205038.428534:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205038.428535:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205038.428537:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205038.428539:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1788205038.428541:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205038.428545:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205038.428547:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788205038.428548:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205038.428551:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205038.428553:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428557:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205038.428562:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205038.428562:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428566:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205038.428571:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205038.428577:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205038.428579:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205038.428582:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.428584:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205038.428586:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205038.428591:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205038.428594:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205038.428596:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205038.428598:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.428602:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428603:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205038.428609:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205038.428611:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543503104:192.168.201.118@tcp:250:kworker.0 00000008:00000001:3.0:1788205038.428616:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.428619:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788205038.428620:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.428626:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1788205038.428627:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788205038.428631:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205038.428632:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205038.428634:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205038.428637:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788205038.428638:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428641:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205038.428643:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1788205038.428645:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:3.0:1788205038.428645:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.428650:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205038.428653:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788205038.428659:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205038.428660:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205038.428667:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18f20. 00000100:00100000:3.0:1788205038.428668:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205038.428677:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1788205038.428679:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205038.428680:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205038.428686:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205038.428696:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205038.428699:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543503104, portal 25 00000100:00080000:3.0:1788205038.428700:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205038.428705:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788205038.428707:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205038.428708:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000001:3.0:1788205038.428710:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205038.428712:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205038.428716:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1788205038.428720:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205038.428722:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26724 00000400:00000200:3.0:1788205038.428730:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205038.428735:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205038.428738:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788205038.428739:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205038.428746:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000040:3.0:1788205038.428746:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428751:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1788205038.428753:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6550. 00000100:00000040:3.0:1788205038.428754:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428759:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205038.428761:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000200:1.0:1788205038.428762:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543503104, offset 0 00000020:00000040:3.0:1788205038.428762:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205038.428765:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205038.428768:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205038.428770:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205038.428774:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205038.428775:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000040:3.0:1788205038.428779:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428782:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205038.428783:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428785:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205038.428787:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:3.0:1788205038.428789:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205038.428793:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428796:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205038.428797:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 02000000:00000001:3.0:1788205038.428798:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205038.428803:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205038.428806:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1788205038.428807:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428810:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205038.428812:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000008:00100000:3.0:1788205038.428813:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205038.428817:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205038.428819:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788205038.428820:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205038.428825:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000010:3.0:1788205038.428825:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205038.428829:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205038.428831:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1788205038.428834:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471279:471279:256:4294967295] 192.168.201.118@tcp LPNI seq info [471138:471138:8:4294967295] 02000000:00000001:3.0:1788205038.428838:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428841:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205038.428843:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1788205038.428844:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205038.428845:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205038.428849:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788205038.428854:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205038.428859:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00100000:3.0:1788205038.428860:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205038.428867:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205038.428868:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205038.428871:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205038.428878:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205038.428879:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205038.428881:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205038.428884:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205038.428885:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:1.0:1788205038.428886:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97100. 00000100:00080000:3.0:1788205038.428887:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205038.428892:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205038.428895:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26724 00000800:00000200:1.0:1788205038.428899:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:3.0:1788205038.428900:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205038.428906:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205038.428911:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1788205038.428911:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:3.0:1788205038.428915:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428917:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205038.428919:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428920:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205038.428921:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205038.428922:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205038.428923:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205038.428925:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205038.428926:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205038.428927:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205038.428928:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428929:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205038.428931:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.428932:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205038.428934:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205038.428936:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205038.428937:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428938:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205038.428940:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205038.428943:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205038.428943:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205038.428946:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.428948:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321398533 00000100:00000001:3.0:1788205038.428951:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000010:1.0:1788205038.428954:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647180. 00000008:00100000:3.0:1788205038.428954:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205038.428958:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428959:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.428963:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000010:3.0:1788205038.428963:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788205038.428968:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205038.428970:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205038.428971:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.428973:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205038.428974:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205038.428975:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321398533 02000000:00000001:3.0:1788205038.428976:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205038.428979:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000010:1.0:1788205038.428983:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647000. 00000100:00000001:3.0:1788205038.428983:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.428990:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00100000:3.0:1788205038.428990:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205038.428998:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205038.428998:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205038.429001:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429006:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205038.429006:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205038.429012:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.429014:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429018:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205038.429018:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205038.429020:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205038.429022:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205038.429025:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205038.429027:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205038.429030:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26724 00000100:00000001:1.0:1788205038.429031:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1788205038.429033:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205038.429035:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:3.0:1788205038.429037:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205038.429040:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205038.429041:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1788205038.429043:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205038.429045:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205038.429046:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205038.429050:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205038.429050:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.429054:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205038.429055:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205038.429056:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205038.429058:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1788205038.429059:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205038.429060:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205038.429063:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205038.429066:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205038.429072:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.429075:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205038.429079:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1788205038.429079:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205038.429080:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205038.429082:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205038.429083:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 02000000:00000001:1.0:1788205038.429087:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788205038.429110:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205038.429114:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205038.429126:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205038.429129:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205038.429131:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.429133:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205038.429136:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205038.429139:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429141:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205038.429142:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429146:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205038.429149:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000100:00100000:1.0:1788205038.429151:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503232:192.168.201.118@tcp:38:kworker.0 00000020:00000040:3.0:1788205038.429154:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788205038.429156:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205038.429159:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205038.429160:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205038.429162:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429163:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205038.429167:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205038.429168:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788205038.429171:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1788205038.429174:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:3.0:1788205038.429175:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429178:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205038.429183:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000400:00000200:1.0:1788205038.429184:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205038.429190:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18790. 00000100:00000001:3.0:1788205038.429190:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205038.429196:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6cc0. 00000100:00100000:3.0:1788205038.429199:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205038.429202:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543503232, portal 10 00000100:00000001:1.0:1788205038.429207:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205038.429210:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000001:3.0:1788205038.429212:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205038.429216:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205038.429221:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205038.429223:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205038.429241:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205038.429247:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205038.429247:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205038.429252:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a65d8. 00000100:00000040:3.0:1788205038.429255:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26734) 00000100:00000200:1.0:1788205038.429256:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543503232, offset 0 00000400:00000200:1.0:1788205038.429264:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205038.429269:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205038.429274:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205038.429278:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205038.429281:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205038.429283:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205038.429285:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205038.429289:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471280:471280:255:4294967295] 192.168.201.118@tcp LPNI seq info [471139:471139:7:4294967295] 00000400:00000200:1.0:1788205038.429293:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205038.429299:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205038.429304:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205038.429307:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97a00. 00000800:00000200:1.0:1788205038.429311:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205038.429315:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788205038.429317:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205038.429321:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205038.429324:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205038.429326:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321397267 00000100:00000001:1.0:1788205038.429329:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321398533 00000100:00000001:1.0:1788205038.429331:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321397267 00000100:00000001:1.0:1788205038.429334:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321398533 00000100:00000001:1.0:1788205038.429336:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205038.429338:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429340:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205038.429344:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205038.429350:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205038.429352:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429354:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205038.429355:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205038.429357:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205038.429360:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503360:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205038.429366:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205038.429368:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205038.429369:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429371:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205038.429374:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205038.429376:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205038.429380:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205038.429382:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18210. 00000400:00000010:1.0:1788205038.429386:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a64c8. 00000100:00000200:1.0:1788205038.429388:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543503360, portal 10 00000100:00000001:1.0:1788205038.429390:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205038.429391:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205038.429397:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205038.429403:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205038.429405:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205038.429407:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6e58. 00000100:00000200:1.0:1788205038.429409:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543503360, offset 0 00000400:00000200:1.0:1788205038.429413:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205038.429417:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205038.429420:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205038.429423:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205038.429426:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205038.429428:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205038.429430:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205038.429433:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471281:471281:254:4294967295] 192.168.201.118@tcp LPNI seq info [471140:471140:6:4294967295] 00000400:00000200:1.0:1788205038.429442:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205038.429448:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205038.429452:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205038.429456:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97200. 00000800:00000200:1.0:1788205038.429459:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205038.429463:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788205038.429466:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205038.429469:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205038.429471:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205038.429473:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321397268 00000100:00000001:1.0:1788205038.429476:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321398533 00000100:00000001:1.0:1788205038.429478:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321397268 00000100:00000001:1.0:1788205038.429480:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321398533 00000100:00000001:1.0:1788205038.429482:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205038.429484:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429486:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205038.429490:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205038.429497:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205038.429499:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429501:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205038.429502:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205038.429504:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205038.429507:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503488:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205038.429510:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205038.429512:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205038.429513:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429515:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205038.429519:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205038.429520:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205038.429524:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205038.429527:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18478. 00000400:00000010:1.0:1788205038.429531:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6a18. 00000100:00000200:1.0:1788205038.429534:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543503488, portal 4 00000100:00000001:1.0:1788205038.429536:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205038.429537:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205038.429543:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205038.429551:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205038.429553:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205038.429555:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6220. 00000100:00000200:1.0:1788205038.429557:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543503488, offset 0 00000400:00000200:1.0:1788205038.429561:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205038.429565:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205038.429568:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205038.429570:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205038.429572:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205038.429575:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205038.429577:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205038.429580:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471282:471282:253:4294967295] 192.168.201.118@tcp LPNI seq info [471141:471141:5:4294967295] 00000400:00000200:1.0:1788205038.429583:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205038.429589:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205038.429593:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205038.429597:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97300. 00000800:00000200:1.0:1788205038.429601:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205038.429604:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788205038.429607:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205038.429610:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205038.429613:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205038.429615:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321397277 00000100:00000001:1.0:1788205038.429617:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321398533 00000100:00000001:1.0:1788205038.429619:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321397277 00000100:00000001:1.0:1788205038.429621:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321398533 00000100:00000001:1.0:1788205038.429623:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205038.429625:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429628:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429630:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.429633:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205038.429635:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788205038.429638:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205038.429639:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205038.429640:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205038.429645:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205038.429651:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205038.429653:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429655:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205038.429656:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205038.429658:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205038.429661:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503616:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205038.429664:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205038.429666:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205038.429667:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205038.429669:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205038.429672:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205038.429674:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205038.429677:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205038.429680:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205038.429683:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205038.429686:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543503616, portal 4 00000100:00000001:1.0:1788205038.429688:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205038.429690:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205038.429696:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205038.429700:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205038.429702:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205038.429705:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6198. 00000100:00000200:1.0:1788205038.429707:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543503616, offset 0 00000400:00000200:1.0:1788205038.429711:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205038.429715:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205038.429718:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205038.429722:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205038.429724:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205038.429726:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205038.429728:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205038.429731:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471283:471283:252:4294967295] 192.168.201.118@tcp LPNI seq info [471142:471142:4:4294967295] 00000400:00000200:1.0:1788205038.429735:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205038.429740:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205038.429745:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205038.429748:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97c00. 00000800:00000200:1.0:1788205038.429752:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205038.429755:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788205038.429758:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205038.429761:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205038.429764:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205038.429766:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321397300 00000100:00000001:1.0:1788205038.429768:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321398533 00000100:00000001:1.0:1788205038.429770:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321397300 00000100:00000001:1.0:1788205038.429772:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321398533 00000100:00000001:1.0:1788205038.429774:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205038.429776:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429778:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429780:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205038.429784:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205038.429785:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788205038.429787:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205038.429788:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205038.429790:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205038.429791:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205039.260243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205039.260249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205039.260255:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205039.260257:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205039.260259:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205039.260261:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205039.260268:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205039.260268:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205039.260274:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205039.260275:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788205039.260278:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205039.260279:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205039.260281:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205039.260284:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205039.260286:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205039.260289:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205039.260292:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205039.260297:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205039.260300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205039.260305:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205039.260306:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205039.260310:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205039.260312:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205039.260384:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205039.260390:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205039.260393:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205039.260845:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205039.260849:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205039.260851:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205039.260854:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205039.260857:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205039.260859:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205039.260861:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205039.260864:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205039.260866:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205039.772194:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321398533, at 4321398592 00000100:00000010:3.0:1788205039.772213:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647000. 00000100:00000040:2.0:1788205039.772303:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321398533, at 4321398592 00000100:00000010:2.0:1788205039.772310:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00000100:00000001:1.0:1788205040.284234:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205040.284241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205040.284245:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205040.284246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205040.284249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205040.284253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205040.284255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205040.284256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205040.284258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205040.284260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205040.284263:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205040.284264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205040.284266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205040.284269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205040.284273:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205040.284277:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205040.284281:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205040.284413:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205040.284417:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205040.284421:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205040.284467:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205040.284474:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205040.284476:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205040.284481:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205040.284486:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205040.284489:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205040.284491:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205040.284494:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205040.284496:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.308233:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205041.308240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205041.308242:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205041.308246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.308246:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205041.308254:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205041.308256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205041.308259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.308260:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205041.308264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205041.308265:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205041.308268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.308272:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205041.308274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205041.308276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205041.308276:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205041.308283:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205041.308590:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205041.308596:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205041.308600:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205041.312367:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205041.312377:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205041.312379:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205041.312387:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205041.312391:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205041.312395:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205041.312397:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205041.312400:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205041.312403:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:0.0:1788205041.500453:0:256123:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:0.0:1788205041.500507:0:256123:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1788205041.500519:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205041.500524:0:256123:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:0.0:1788205041.500531:0:256123:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205041.500536:0:256123:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:0.0:1788205041.500541:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788205041.500546:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97100. 00000400:00000200:0.0:1788205041.500553:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788205041.500558:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788205041.500566:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6550 00000400:00000010:0.0:1788205041.500570:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6550. 00000100:00000001:0.0:1788205041.500576:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205041.500587:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.500614:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205041.500619:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.500634:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.500636:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788205041.500648:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788205041.500651:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97a00. 00000400:00000200:0.0:1788205041.500655:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1788205041.500658:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1788205041.500662:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a65d8 00000400:00000010:0.0:1788205041.500665:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a65d8. 00000100:00000001:0.0:1788205041.500668:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1788205041.500673:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.500675:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205041.500682:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205041.500686:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788205041.500687:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1788205041.500692:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205041.500700:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205038/real 1788205041] req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.500702:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.500705:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1788205041.500714:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788205041.500718:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97200. 00000400:00000200:0.0:1788205041.500720:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000001:1.0:1788205041.500722:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:0.0:1788205041.500725:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000001:1.0:1788205041.500726:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:0.0:1788205041.500729:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6e58 00000400:00000010:0.0:1788205041.500733:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6e58. 00000400:00000010:1.0:1788205041.500733:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18790. 00000100:00000001:0.0:1788205041.500738:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:1.0:1788205041.500742:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6cc0 00000100:00000200:0.0:1788205041.500744:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788205041.500746:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6cc0. 00000100:00000001:1.0:1788205041.500752:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1788205041.500757:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1788205041.500762:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205041.500765:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.500784:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.500787:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:1.0:1788205041.500788:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1788205041.500797:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788205041.500803:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97300. 00000100:00000001:1.0:1788205041.500804:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:0.0:1788205041.500807:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1788205041.500808:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788205041.500813:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1788205041.500814:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:0.0:1788205041.500817:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6220 00000100:00000001:1.0:1788205041.500818:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1788205041.500820:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6220. 00000100:00000001:0.0:1788205041.500825:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1788205041.500829:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788205041.500832:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.500852:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205041.500857:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.500861:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205041.500869:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.500870:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1788205041.500879:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1788205041.500887:0:256123:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:0.0:1788205041.500892:0:256123:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97c00. 00000100:00000001:1.0:1788205041.500894:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:0.0:1788205041.500897:0:256123:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1788205041.500900:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1788205041.500902:0:256123:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1788205041.500904:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:0.0:1788205041.500906:0:256123:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6198 00000100:00000001:1.0:1788205041.500907:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:0.0:1788205041.500911:0:256123:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6198. 00000100:00000001:1.0:1788205041.500913:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788205041.500914:0:256123:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1788205041.500919:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000200:0.0:1788205041.500920:0:256123:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205041.500927:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788205041.500936:0:256123:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1788205041.500938:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000040:0.0:1788205041.500942:0:256123:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205063 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.500944:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205041.500954:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205041.500955:0:256123:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205041.500958:0:256123:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1788205041.500968:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503232:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205041.500975:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205041.500979:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205038/real 1788205041] req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205041.500988:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205041.500989:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205041.500991:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18210. 00000400:00000200:1.0:1788205041.500994:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a64c8 00000400:00000010:1.0:1788205041.500996:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a64c8. 00000100:00000001:1.0:1788205041.500999:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205041.501002:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205041.501016:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501023:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205041.501025:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501027:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205041.501028:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205041.501032:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501039:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205041.501042:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501050:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205041.501052:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501054:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205041.501055:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501057:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205041.501060:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205041.501076:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205041.501081:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205041.501083:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205041.501087:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205041.501118:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503360:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205041.501123:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205041.501127:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205038/real 1788205041] req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205041.501142:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205041.501144:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205041.501146:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18478. 00000400:00000200:1.0:1788205041.501149:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6a18 00000400:00000010:1.0:1788205041.501151:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6a18. 00000100:00000001:1.0:1788205041.501154:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205041.501158:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205041.501168:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501174:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205041.501176:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501178:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205041.501179:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205041.501183:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501190:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205041.501194:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501201:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205041.501203:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501205:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205041.501206:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501208:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205041.501211:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205041.501213:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205041.501217:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205041.501220:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205041.501226:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205041.501235:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503488:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205041.501240:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205041.501243:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205038/real 1788205041] req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205041.501252:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205041.501253:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205041.501255:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18f20. 00000400:00000200:1.0:1788205041.501258:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205041.501260:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205041.501263:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205041.501266:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205041.501275:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501282:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205041.501284:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501287:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205041.501288:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205041.501293:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501301:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205041.501305:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501312:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205041.501314:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501316:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205041.501461:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501468:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205041.501471:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205041.501477:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205041.501488:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205041.501492:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205041.501502:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205041.501514:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543503104:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205041.501521:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205041.501525:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205041.501530:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543503232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501538:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205041.501542:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205041.501546:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205041.501553:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205041.501556:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205041.501559:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205041.501562:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205041.501564:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205041.501569:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205041.501571:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205041.501573:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205041.501577:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205041.501578:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501580:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205041.501584:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543503360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501591:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205041.501592:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205041.501593:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205041.501597:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205041.501598:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205041.501599:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205041.501602:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205041.501603:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205041.501606:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205041.501607:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205041.501608:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205041.501611:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205041.501612:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501614:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205041.501618:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543503488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501630:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205041.501632:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205041.501633:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205041.501636:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205041.501638:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205041.501639:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205041.501641:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205041.501642:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205041.501644:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205041.501645:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205041.501647:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205041.501649:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205041.501650:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501652:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205041.501655:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543503104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501661:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205041.501663:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205041.501664:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205041.501666:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205041.501667:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205041.501668:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205041.501670:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205041.501672:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205041.501673:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205041.501674:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205041.501676:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205041.501679:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205041.501680:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501682:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205041.501685:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205041.501687:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205041.501690:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205041.501695:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205038/real 1788205041] req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205041.501711:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205041.501713:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205041.501716:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205041.501720:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205041.501723:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205041.501740:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205041.501745:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205041.501755:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501761:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205041.501763:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501765:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205041.501766:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205041.501770:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501775:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205041.501779:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501786:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205041.501790:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501791:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205041.501792:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501793:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205041.501795:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205041.501797:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205041.501801:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205041.501803:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205041.501806:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205041.501814:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503616:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205041.501818:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501820:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205041.501822:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543503616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205063 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205041.501827:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205041.501829:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205041.501830:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205041.501832:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205041.501834:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205041.501835:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205041.501836:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205041.501838:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205041.501840:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205041.501841:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205041.501843:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205041.501845:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205041.501846:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501848:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205041.501857:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205041.501858:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205041.501861:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205041.501863:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205042.332180:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205042.332187:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205042.332190:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205042.332194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205042.332198:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205042.332199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205042.332203:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205042.332208:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205042.332210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205042.332211:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205042.332213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205042.332217:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205042.332218:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205042.332219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205042.332221:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205042.332225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205042.332229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205042.332232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205042.332234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205042.332236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205042.332244:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205042.332251:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205042.332255:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205042.332257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205042.332261:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205042.332264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205042.332300:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205042.332303:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205042.332305:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205043.356221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205043.356221:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205043.356228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205043.356230:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205043.356231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205043.356234:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205043.356235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205043.356235:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205043.356241:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205043.356243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205043.356245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205043.356246:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205043.356248:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205043.356249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.356250:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205043.356252:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205043.356255:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205043.356257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205043.356257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205043.356258:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205043.356261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.356263:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205043.356267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205043.356270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205043.356270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205043.356275:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205043.356354:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205043.356358:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205043.356362:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205043.548242:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205043.548257:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205043.548261:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205043.548265:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205043.548270:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205043.548280:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26729 00000400:00000200:3.0:1788205043.548292:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205043.548300:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205043.548304:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205043.548308:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548311:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.548315:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548318:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205043.548322:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205043.548325:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205043.548328:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205043.548332:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205043.548335:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205043.548338:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205043.548341:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548343:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.548357:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548360:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205043.548363:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205043.548366:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548369:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205043.548372:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205043.548375:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205043.548383:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.548386:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205043.548389:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205043.548397:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205043.548401:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205043.548406:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205043.548408:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.548412:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205043.548415:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205043.548420:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205043.548429:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412000. 00000100:00000001:3.0:1788205043.548436:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205043.548448:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205043.548460:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205043.548466:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205043.548834:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205043.548874:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205043.548878:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205043.548882:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205043.548883:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205043.548885:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205043.548888:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205043.548892:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26729 00000400:00000200:3.0:1788205043.548896:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205043.548900:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205043.548902:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205043.548904:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548906:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.548908:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548910:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205043.548913:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205043.548915:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205043.548916:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205043.548919:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205043.548921:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205043.548923:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205043.548926:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548929:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.548930:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548932:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205043.548934:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205043.548936:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205043.548942:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205043.548944:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205043.548951:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205043.548953:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.548955:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205043.548961:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205043.548964:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.548966:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205043.548971:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788205043.548973:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205043.548976:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205043.548977:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.548979:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205043.548980:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205043.548982:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205043.548986:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411c00. 00000100:00000001:3.0:1788205043.548990:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205043.549083:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205043.549123:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205043.549125:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205043.549129:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205043.549138:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205043.549141:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205043.549145:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205043.549147:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205043.549148:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205043.549151:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205043.549154:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26729 00000400:00000200:3.0:1788205043.549159:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205043.549162:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205043.549165:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205043.549167:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549169:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.549171:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549173:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205043.549175:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205043.549176:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205043.549177:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205043.549179:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205043.549181:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205043.549183:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205043.549185:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549187:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.549189:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549191:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205043.549193:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205043.549196:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549198:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205043.549200:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205043.549203:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205043.549205:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549207:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205043.549209:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205043.549212:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549213:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205043.549217:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205043.549219:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205043.549222:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205043.549223:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549225:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205043.549228:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205043.549229:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205043.549233:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412c00. 00000100:00000001:3.0:1788205043.549237:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205043.549242:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205043.549249:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205043.549251:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205043.549254:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205043.549261:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205043.549264:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205043.549267:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205043.549269:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205043.549270:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205043.549273:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205043.549276:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26729 00000400:00000200:3.0:1788205043.549279:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205043.549282:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205043.549284:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205043.549287:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549289:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.549291:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549293:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205043.549295:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205043.549297:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205043.549298:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205043.549300:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205043.549302:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205043.549304:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205043.549308:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549310:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.549312:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549314:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205043.549317:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205043.549320:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549322:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205043.549324:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205043.549327:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205043.549329:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549330:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205043.549333:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205043.549335:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549336:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205043.549340:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788205043.549342:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205043.549344:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205043.549357:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549359:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205043.549360:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205043.549362:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205043.549368:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788205043.549372:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205043.549377:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205043.549383:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205043.549385:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205043.549389:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205043.549402:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205043.549404:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205043.549408:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205043.549410:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205043.549411:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205043.549414:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205043.549416:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26729 00000400:00000200:3.0:1788205043.549420:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205043.549423:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205043.549425:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205043.549428:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549430:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.549431:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549433:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205043.549436:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205043.549437:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205043.549439:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205043.549441:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205043.549443:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205043.549445:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205043.549449:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549450:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205043.549452:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549454:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205043.549456:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205043.549459:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205043.549461:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205043.549463:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205043.549466:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205043.549468:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549470:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205043.549472:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205043.549474:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549476:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205043.549492:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205043.549495:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205043.549498:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205043.549499:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205043.549501:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205043.549502:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205043.549504:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205043.549508:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:3.0:1788205043.549511:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205043.549516:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205043.549522:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205043.549524:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205043.549528:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205043.549534:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205043.549539:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26740) 00000100:00000001:1.0:1788205043.551269:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205043.551280:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205043.551284:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205043.551297:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551310:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205043.551312:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.551318:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205043.551322:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205043.551325:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205043.551329:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543503744:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205043.551334:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205043.551340:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205043.551343:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.551346:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205043.551355:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205043.551357:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205043.551364:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205043.551368:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205043.551376:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205043.551384:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543503744, portal 25 00000100:00000001:1.0:1788205043.551387:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205043.551388:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205043.551411:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551420:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205043.551423:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205043.551426:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205043.551431:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543503744, offset 0 00000400:00000200:1.0:1788205043.551439:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205043.551450:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.551459:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205043.551465:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205043.551471:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205043.551484:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205043.551488:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205043.551493:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471284:471284:256:4294967295] 192.168.201.118@tcp LPNI seq info [471143:471143:8:4294967295] 00000400:00000200:1.0:1788205043.551498:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.551508:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205043.551515:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.551522:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205043.551530:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205043.551534:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205043.551540:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205043.551544:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205043.551549:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.551551:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205043.551559:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205043.551563:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205043.551570:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205043.551573:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205043.551577:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205043.551582:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551592:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.551595:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551601:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.551603:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205043.551610:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205043.551614:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205043.551617:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321403656 00000100:00000010:1.0:1788205043.551623:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647c60. 00000100:00000001:1.0:1788205043.551630:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205043.551638:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321403656 00000100:00000010:1.0:1788205043.551642:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647420. 00000100:00000001:1.0:1788205043.551645:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205043.551648:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205043.551651:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.551654:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205043.551658:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205043/real 1788205043] req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205043.551668:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205043.551669:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205043.551672:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205043.551675:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205043.551676:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205043.551679:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205043.551683:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205043.551691:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551698:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205043.551700:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.551702:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.551704:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205043.551708:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551716:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205043.551721:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551727:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205043.551731:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.551732:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.551734:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.551737:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205043.551740:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205043.551743:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205043.551748:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205043.551751:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205043.551755:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205043.551760:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543503744:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205043.551764:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205043.551767:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551771:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205043.551772:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.551774:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205043.551775:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205043.551776:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205043.551778:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503872:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205043.551780:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205043.551783:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205043.551784:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.551785:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205043.551788:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205043.551789:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205043.551792:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205043.551793:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205043.551797:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205043.551799:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543503872, portal 10 00000100:00000001:1.0:1788205043.551800:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205043.551801:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205043.551806:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551811:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205043.551812:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205043.551813:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205043.551815:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543503872, offset 0 00000400:00000200:1.0:1788205043.551817:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205043.551820:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.551822:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205043.551824:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205043.551826:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205043.551828:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205043.551829:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205043.551831:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471285:471285:256:4294967295] 192.168.201.118@tcp LPNI seq info [471144:471144:8:4294967295] 00000400:00000200:1.0:1788205043.551834:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.551838:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205043.551841:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.551843:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205043.551845:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205043.551847:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205043.551850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205043.551852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205043.551854:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.551856:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205043.551858:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205043.551860:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205043.551862:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205043.551863:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205043.551864:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205043.551866:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551871:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.551873:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551881:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.551882:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205043.551884:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205043.551887:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205043.551889:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205043.551890:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321402389 00000100:00000001:1.0:1788205043.551892:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321403656 00000100:00000001:1.0:1788205043.551894:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321402389 00000100:00000001:1.0:1788205043.551895:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321403656 00000100:00000001:1.0:1788205043.551897:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205043.551898:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.551899:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205043.551902:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205043/real 1788205043] req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205043.551909:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205043.551911:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205043.551912:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205043.551914:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205043.551916:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205043.551918:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205043.551921:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205043.551928:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551935:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205043.551936:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.551938:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.551939:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205043.551943:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551951:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205043.551955:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551961:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205043.551962:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.551964:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.551965:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.551966:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205043.551968:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205043.551969:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205043.551971:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205043.551973:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205043.551975:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205043.551980:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543503872:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205043.551982:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205043.551984:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.551988:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205043.551990:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.551991:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205043.551992:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205043.551994:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205043.551995:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504000:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205043.551997:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205043.551999:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205043.552001:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.552002:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205043.552004:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205043.552005:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205043.552008:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205043.552009:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205043.552012:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205043.552014:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543504000, portal 10 00000100:00000001:1.0:1788205043.552015:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205043.552016:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205043.552020:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552024:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205043.552038:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205043.552041:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205043.552044:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543504000, offset 0 00000400:00000200:1.0:1788205043.552048:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205043.552052:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.552056:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205043.552059:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205043.552061:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205043.552063:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205043.552066:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205043.552068:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471286:471286:256:4294967295] 192.168.201.118@tcp LPNI seq info [471145:471145:8:4294967295] 00000400:00000200:1.0:1788205043.552076:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.552082:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205043.552087:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.552138:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205043.552142:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205043.552145:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205043.552149:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205043.552151:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205043.552155:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.552158:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205043.552161:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205043.552164:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205043.552167:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205043.552168:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205043.552171:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205043.552174:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552181:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.552184:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552189:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.552190:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205043.552193:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205043.552196:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205043.552199:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205043.552201:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321402391 00000100:00000001:1.0:1788205043.552203:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321403656 00000100:00000001:1.0:1788205043.552205:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321402391 00000100:00000001:1.0:1788205043.552207:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321403656 00000100:00000001:1.0:1788205043.552209:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205043.552211:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.552213:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205043.552216:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205043/real 1788205043] req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205043.552224:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205043.552225:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205043.552226:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205043.552229:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205043.552230:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205043.552233:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205043.552236:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205043.552244:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552250:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205043.552252:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.552254:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.552255:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205043.552258:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552264:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205043.552268:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552275:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205043.552277:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.552279:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.552280:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.552282:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205043.552285:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205043.552287:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205043.552291:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205043.552293:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205043.552298:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205043.552305:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504000:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205043.552308:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205043.552313:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552319:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205043.552321:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.552323:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205043.552325:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205043.552327:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205043.552330:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504128:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205043.552333:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205043.552336:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205043.552338:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.552340:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205043.552343:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205043.552345:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205043.552348:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205043.552350:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205043.552354:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205043.552357:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543504128, portal 4 00000100:00000001:1.0:1788205043.552359:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205043.552360:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205043.552366:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552372:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205043.552375:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205043.552377:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205043.552379:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543504128, offset 0 00000400:00000200:1.0:1788205043.552383:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205043.552387:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.552390:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205043.552393:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205043.552396:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205043.552397:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205043.552401:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205043.552404:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471287:471287:256:4294967295] 192.168.201.118@tcp LPNI seq info [471146:471146:8:4294967295] 00000400:00000200:1.0:1788205043.552407:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.552413:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205043.552416:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.552419:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205043.552422:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205043.552425:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205043.552428:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205043.552431:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205043.552434:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.552436:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205043.552439:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205043.552442:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205043.552444:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205043.552446:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205043.552448:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205043.552451:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552458:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.552461:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552466:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.552468:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205043.552470:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205043.552473:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205043.552476:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205043.552478:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321402400 00000100:00000001:1.0:1788205043.552480:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321403656 00000100:00000001:1.0:1788205043.552487:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321402400 00000100:00000001:1.0:1788205043.552490:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321403656 00000100:00000001:1.0:1788205043.552492:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205043.552495:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.552496:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205043.552500:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205043/real 1788205043] req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205043.552507:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205043.552508:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205043.552510:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205043.552512:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205043.552514:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205043.552516:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205043.552519:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205043.552527:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552533:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205043.552535:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.552536:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.552537:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205043.552539:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552544:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205043.552546:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552550:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205043.552551:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.552552:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.552553:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.552554:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205043.552555:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205043.552556:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205043.552559:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205043.552561:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205043.552563:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205043.552567:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504128:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205043.552569:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205043.552572:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552575:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205043.552576:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.552577:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205043.552578:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205043.552580:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205043.552581:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504256:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205043.552583:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205043.552585:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205043.552586:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205043.552587:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205043.552589:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205043.552591:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205043.552592:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205043.552594:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205043.552596:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205043.552598:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543504256, portal 4 00000100:00000001:1.0:1788205043.552599:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205043.552600:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205043.552604:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552609:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205043.552721:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205043.552732:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205043.552740:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543504256, offset 0 00000400:00000200:1.0:1788205043.552747:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205043.552758:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.552767:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205043.552772:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205043.552776:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205043.552780:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205043.552782:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205043.552793:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471288:471288:256:4294967295] 192.168.201.118@tcp LPNI seq info [471147:471147:8:4294967295] 00000400:00000200:1.0:1788205043.552799:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205043.552808:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205043.552815:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.552821:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205043.552827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205043.552831:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205043.552837:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205043.552842:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205043.552846:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205043.552850:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205043.552855:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205043.552860:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205043.552865:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205043.552867:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205043.552872:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205043.552879:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552892:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.552895:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205068 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552903:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.552905:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205043.552911:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205043.552916:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205043.552920:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205043.552924:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321402423 00000100:00000001:1.0:1788205043.552927:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321403656 00000100:00000001:1.0:1788205043.552929:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321402423 00000100:00000001:1.0:1788205043.552932:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321403656 00000100:00000001:1.0:1788205043.552934:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205043.552937:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.552940:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205043.552945:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205043/real 1788205043] req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205043.552955:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205043.552957:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205043.552960:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205043.552963:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205043.552965:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205043.552967:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205043.552970:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205043.552979:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.552986:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205043.552987:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.552989:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.552991:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205043.552995:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.553001:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205043.553005:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.553014:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205043.553017:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553018:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205043.553020:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553023:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205043.553025:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205043.553049:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205043.553054:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205043.553056:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205043.553060:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205043.553069:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504256:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205043.553074:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553077:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.553081:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543503744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.553088:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205043.553131:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205043.553136:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205043.553140:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205043.553143:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205043.553145:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205043.553147:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205043.553149:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788205043.553153:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205043.553156:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205043.553158:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205043.553162:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205043.553163:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553166:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.553170:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543503872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.553177:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205043.553179:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205043.553180:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205043.553182:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205043.553183:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205043.553185:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205043.553186:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205043.553188:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788205043.553190:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205043.553190:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205043.553191:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205043.553194:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205043.553195:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553196:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.553200:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543504000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.553206:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205043.553215:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205043.553217:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205043.553220:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205043.553221:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205043.553222:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205043.553224:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205043.553225:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:1.0:1788205043.553227:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205043.553228:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205043.553229:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205043.553231:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205043.553233:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553234:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.553238:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543504128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.553249:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205043.553250:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205043.553251:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205043.553253:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205043.553254:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205043.553255:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205043.553257:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205043.553258:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205043.553260:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205043.553261:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205043.553262:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205043.553264:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205043.553265:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553266:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205043.553269:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543504256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205068 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205043.553274:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205043.553276:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205043.553277:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205043.553279:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205043.553280:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205043.553281:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205043.553282:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205043.553284:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205043.553286:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205043.553287:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205043.553288:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205043.553290:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205043.553291:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553293:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205043.553300:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205043.553302:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205043.553305:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205043.553307:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205044.380228:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205044.380228:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205044.380232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205044.380238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205044.380239:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205044.380239:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205044.380244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205044.380244:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205044.380248:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205044.380250:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205044.380252:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205044.380252:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205044.380255:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205044.380256:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205044.380257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205044.380261:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205044.380262:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205044.380262:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205044.380266:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205044.380269:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788205044.380269:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205044.380270:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205044.380274:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205044.380278:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380279:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380283:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380286:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205044.380293:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205044.380295:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205044.380297:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380298:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205044.380299:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380301:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380303:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205044.380308:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205044.380310:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205044.380311:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380313:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205044.380316:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380318:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205044.380319:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205044.380350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205044.380354:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205044.380357:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205044.893119:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321403656, at 4321403713 00000100:00000010:3.0:1788205044.893142:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647420. 00000100:00000040:2.0:1788205044.893241:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321403656, at 4321403713 00000100:00000010:2.0:1788205044.893248:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00000100:00000001:2.0:1788205045.404226:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205045.404233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205045.404243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205045.404244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205045.404246:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205045.404250:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205045.404251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205045.404253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205045.404254:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205045.404256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205045.404257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205045.404257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205045.404260:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205045.404261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205045.404262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205045.404263:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205045.404266:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205045.404378:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205045.404384:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205045.404388:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205045.404434:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205045.404439:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205045.404440:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205045.404442:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205045.404445:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205045.404446:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205045.404449:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205045.404452:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205045.404454:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205046.428180:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205046.428181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205046.428187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205046.428187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205046.428189:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205046.428193:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205046.428194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205046.428194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205046.428197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205046.428198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205046.428199:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205046.428200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205046.428202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205046.428204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205046.428205:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205046.428206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205046.428209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205046.428211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205046.428212:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205046.428215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205046.428216:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205046.428220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205046.428221:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205046.428224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205046.428226:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205046.428227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205046.428234:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205046.428237:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205046.428239:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205047.452187:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205047.452189:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205047.452191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205047.452197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205047.452197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205047.452197:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205047.452202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205047.452202:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205047.452205:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205047.452207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205047.452209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205047.452210:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205047.452212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205047.452215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205047.452216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205047.452216:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205047.452220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205047.452222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205047.452223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205047.452223:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205047.452229:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205047.452230:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205047.452234:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205047.452235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205047.452238:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205047.452239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205047.452345:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205047.452348:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205047.452352:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205048.476168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205048.476168:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205048.476168:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205048.476175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205048.476176:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205048.476180:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205048.476182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205048.476184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205048.476185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205048.476185:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205048.476188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205048.476189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205048.476191:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205048.476193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205048.476194:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205048.476196:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205048.476197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205048.476198:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205048.476201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.476203:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205048.476203:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205048.476207:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205048.476208:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.476210:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.476215:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205048.476215:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205048.476264:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205048.476268:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205048.476270:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205048.668152:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205048.668160:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205048.668162:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205048.668166:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205048.668169:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205048.668174:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26735 00000400:00000200:3.0:1788205048.668182:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205048.668186:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205048.668188:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205048.668191:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668193:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205048.668195:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668196:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205048.668199:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205048.668200:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205048.668202:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205048.668205:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205048.668207:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205048.668208:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205048.668210:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668211:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205048.668213:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668214:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205048.668215:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205048.668217:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668219:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205048.668221:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205048.668223:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205048.668225:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668226:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205048.668228:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205048.668235:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205048.668238:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205048.668241:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205048.668242:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668245:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205048.668247:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205048.668250:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205048.668269:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788205048.668277:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205048.668286:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205048.668296:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205048.668301:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205048.668305:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205048.668327:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205048.668330:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205048.668332:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205048.668333:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205048.668334:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205048.668336:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205048.668338:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26735 00000400:00000200:3.0:1788205048.668341:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205048.668343:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205048.668344:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205048.668347:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668348:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205048.668350:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668352:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205048.668354:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205048.668355:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205048.668356:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205048.668358:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205048.668359:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205048.668360:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205048.668362:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1788205048.668363:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668366:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205048.668367:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1788205048.668368:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205048.668371:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788205048.668371:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205048.668373:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205048.668378:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000040:1.0:1788205048.668380:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205048.668380:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205048.668384:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205048.668390:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788205048.668390:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205048.668393:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205048.668394:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.668397:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205048.668397:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205048.668400:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788205048.668401:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1788205048.668403:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.668404:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668406:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205048.668409:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543504384:192.168.201.118@tcp:250:kworker.0 00000100:00000010:3.0:1788205048.668410:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205048.668415:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1788205048.668415:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205048.668419:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205048.668421:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205048.668421:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.668425:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668425:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205048.668427:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205048.668429:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788205048.668430:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205048.668434:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000010:3.0:1788205048.668435:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205048.668438:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668440:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205048.668444:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:3.0:1788205048.668447:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1788205048.668450:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00000001:3.0:1788205048.668456:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1788205048.668457:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205048.668459:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205048.668463:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205048.668464:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543504384, portal 25 00000100:00000001:1.0:1788205048.668469:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205048.668471:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000001:3.0:1788205048.668474:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205048.668478:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205048.668478:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205048.668482:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205048.668485:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205048.668487:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205048.668492:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788205048.668492:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205048.668496:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26735 00000100:00000040:1.0:1788205048.668500:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205048.668502:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205048.668505:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000400:00000200:3.0:1788205048.668506:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205048.668510:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1788205048.668513:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543504384, offset 0 00000100:00000040:3.0:1788205048.668513:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668517:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205048.668521:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668525:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205048.668526:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000020:00000001:3.0:1788205048.668531:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205048.668533:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205048.668534:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205048.668537:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000200:1.0:1788205048.668538:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:3.0:1788205048.668542:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205048.668546:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205048.668549:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000040:3.0:1788205048.668554:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205048.668557:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788205048.668558:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205048.668564:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000040:3.0:1788205048.668565:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668569:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205048.668571:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1788205048.668572:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:1.0:1788205048.668577:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00080000:3.0:1788205048.668578:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668581:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205048.668583:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205048.668587:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471289:471289:256:4294967295] 192.168.201.118@tcp LPNI seq info [471148:471148:8:4294967295] 02000000:00000001:3.0:1788205048.668587:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205048.668592:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668595:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205048.668599:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00100000:3.0:1788205048.668599:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205048.668603:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668604:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205048.668607:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205048.668609:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205048.668612:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205048.668614:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668617:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1788205048.668618:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:3.0:1788205048.668619:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205048.668622:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205048.668626:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000800:00000200:1.0:1788205048.668629:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668630:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205048.668636:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1788205048.668637:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00000001:3.0:1788205048.668645:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205048.668648:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205048.668649:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:3.0:1788205048.668653:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1788205048.668655:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1788205048.668663:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205048.668668:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1788205048.668668:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:3.0:1788205048.668675:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205048.668676:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.1:1788205048.668676:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00080000:3.0:1788205048.668679:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205048.668683:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:1.0:1788205048.668687:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205048.668688:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26735 00000800:00000010:1.0:1788205048.668692:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:3.0:1788205048.668693:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205048.668697:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205048.668699:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1788205048.668700:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1788205048.668702:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668706:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205048.668709:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205048.668712:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788205048.668712:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205048.668717:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205048.668718:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205048.668720:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1788205048.668721:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000020:00000040:3.0:1788205048.668722:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000400:00000010:1.0:1788205048.668726:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000020:00000001:3.0:1788205048.668726:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205048.668727:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205048.668732:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668734:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205048.668735:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1788205048.668736:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668739:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205048.668741:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000200:1.0:1788205048.668744:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205048.668744:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205048.668748:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205048.668749:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205048.668750:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205048.668751:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668752:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205048.668754:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205048.668757:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668758:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205048.668760:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788205048.668763:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205048.668767:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205048.668769:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668770:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205048.668771:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788205048.668772:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205048.668774:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205048.668777:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788205048.668778:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:3.0:1788205048.668782:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205048.668789:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205048.668790:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.668794:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205048.668798:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205048.668800:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668800:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205048.668807:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205048.668810:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205048.668815:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321408773 00000100:00000001:3.0:1788205048.668817:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205048.668819:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:1.0:1788205048.668820:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647540. 00000100:00080000:3.0:1788205048.668821:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205048.668824:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205048.668826:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205048.668828:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205048.668828:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205048.668831:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26735 00000400:00000200:3.0:1788205048.668835:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205048.668836:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321408773 00000400:00000200:3.0:1788205048.668839:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000010:1.0:1788205048.668840:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16471e0. 00000100:00080000:3.0:1788205048.668842:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205048.668844:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000040:3.0:1788205048.668846:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205048.668848:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205048.668849:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205048.668853:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205048.668854:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205048.668858:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788205048.668858:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205048.668862:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205048.668863:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00100000:1.0:1788205048.668865:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205048/real 1788205048] req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205048.668866:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205048.668870:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205048.668875:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205048.668881:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000001:1.0:1788205048.668889:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205048.668891:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1788205048.668891:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000010:1.0:1788205048.668894:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000100:00000001:3.0:1788205048.668896:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205048.668897:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205048.668899:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205048.668901:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205048.668903:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205048.668911:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000200:1.0:1788205048.668913:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205048.668917:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205048.668923:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:1.0:1788205048.668929:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1788205048.668930:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205048.668932:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205048.668935:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.668937:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.668954:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205048.668960:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205048.668961:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205048.668966:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205048.668971:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788205048.668971:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205048.668977:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:1.0:1788205048.668979:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788205048.668980:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205048.668988:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1788205048.668988:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.668994:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205048.668994:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.668998:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.668999:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669001:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000010:3.0:1788205048.669002:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205048.669003:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:3.0:1788205048.669006:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00080000:1.0:1788205048.669007:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1788205048.669011:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00080000:1.0:1788205048.669013:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 02000000:00000001:3.0:1788205048.669014:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669017:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788205048.669018:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205048.669020:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1788205048.669023:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205048.669024:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205048.669030:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00100000:1.0:1788205048.669035:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543504384:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205048.669035:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669041:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205048.669041:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205048.669043:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669048:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543504384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205048.669054:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205048.669058:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669059:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205048.669062:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1788205048.669063:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788205048.669065:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205048.669069:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205048.669071:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000001:3.0:1788205048.669073:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205048.669074:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205048.669078:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000100:00000040:3.0:1788205048.669080:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26745) 02000000:00000010:1.0:1788205048.669081:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205048.669086:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205048.669087:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205048.669088:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205048.669114:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205048.669115:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669117:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669124:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205048.669125:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669128:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205048.669129:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205048.669130:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205048.669135:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669141:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205048.669142:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.669144:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205048.669146:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205048.669148:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205048.669151:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504512:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205048.669155:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205048.669156:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205048.669158:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.669160:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205048.669163:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205048.669165:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205048.669169:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205048.669170:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205048.669173:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205048.669175:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543504512, portal 10 00000100:00000001:1.0:1788205048.669177:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205048.669177:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205048.669182:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669186:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205048.669188:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205048.669189:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205048.669190:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543504512, offset 0 00000400:00000200:1.0:1788205048.669193:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205048.669195:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205048.669198:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205048.669200:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205048.669202:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205048.669203:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205048.669205:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205048.669206:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471290:471290:256:4294967295] 192.168.201.118@tcp LPNI seq info [471149:471149:8:4294967295] 00000400:00000200:1.0:1788205048.669209:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205048.669213:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205048.669216:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205048.669218:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205048.669220:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205048.669222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205048.669225:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205048.669227:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205048.669229:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205048.669230:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205048.669232:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205048.669234:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205048.669237:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205048.669238:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205048.669240:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205048.669243:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669250:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669253:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669259:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669260:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205048.669272:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205048.669275:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205048.669279:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205048.669281:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321407507 00000100:00000001:1.0:1788205048.669283:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321408773 00000100:00000001:1.0:1788205048.669285:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321407507 00000100:00000001:1.0:1788205048.669287:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321408773 00000100:00000001:1.0:1788205048.669288:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205048.669290:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669292:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205048.669296:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205048/real 1788205048] req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205048.669304:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205048.669305:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205048.669308:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205048.669310:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205048.669312:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205048.669314:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205048.669317:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205048.669325:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669332:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205048.669333:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669334:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.669335:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205048.669338:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669342:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205048.669344:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669348:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205048.669349:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669350:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.669351:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669352:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205048.669353:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205048.669354:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205048.669356:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205048.669357:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205048.669359:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205048.669364:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504512:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205048.669366:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205048.669368:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669372:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205048.669373:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.669374:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205048.669375:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205048.669377:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205048.669378:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504640:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205048.669380:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205048.669382:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205048.669383:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.669384:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205048.669386:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205048.669388:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205048.669390:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205048.669391:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205048.669393:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205048.669395:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543504640, portal 10 00000100:00000001:1.0:1788205048.669396:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205048.669397:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205048.669400:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669404:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205048.669406:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205048.669407:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205048.669408:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543504640, offset 0 00000400:00000200:1.0:1788205048.669410:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205048.669413:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205048.669415:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205048.669417:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205048.669419:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205048.669420:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205048.669422:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205048.669424:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471291:471291:256:4294967295] 192.168.201.118@tcp LPNI seq info [471150:471150:8:4294967295] 00000400:00000200:1.0:1788205048.669426:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205048.669430:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205048.669432:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205048.669434:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205048.669436:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205048.669438:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205048.669440:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205048.669442:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205048.669444:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205048.669445:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205048.669447:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205048.669449:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205048.669451:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205048.669452:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205048.669453:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205048.669455:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669460:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669461:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669465:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669466:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205048.669468:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205048.669469:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205048.669471:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205048.669473:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321407508 00000100:00000001:1.0:1788205048.669474:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321408773 00000100:00000001:1.0:1788205048.669476:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321407508 00000100:00000001:1.0:1788205048.669478:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321408773 00000100:00000001:1.0:1788205048.669479:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205048.669480:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669481:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205048.669483:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205048/real 1788205048] req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205048.669487:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205048.669488:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205048.669489:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205048.669490:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205048.669491:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205048.669492:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205048.669494:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205048.669502:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669507:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205048.669508:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669510:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.669511:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205048.669515:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669521:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205048.669525:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669532:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205048.669534:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669536:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.669537:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669539:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205048.669541:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205048.669543:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205048.669545:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205048.669546:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205048.669549:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205048.669553:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504640:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205048.669555:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205048.669557:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669560:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205048.669561:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.669563:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205048.669564:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205048.669565:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205048.669567:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504768:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205048.669569:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205048.669570:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205048.669571:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.669572:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205048.669574:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205048.669575:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205048.669577:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205048.669579:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205048.669581:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205048.669583:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543504768, portal 4 00000100:00000001:1.0:1788205048.669584:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205048.669585:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788205048.669588:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669592:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205048.669594:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205048.669595:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205048.669596:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543504768, offset 0 00000400:00000200:1.0:1788205048.669598:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205048.669601:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205048.669603:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205048.669605:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205048.669606:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205048.669607:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205048.669609:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205048.669611:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471292:471292:256:4294967295] 192.168.201.118@tcp LPNI seq info [471151:471151:8:4294967295] 00000400:00000200:1.0:1788205048.669613:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205048.669617:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205048.669620:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205048.669622:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205048.669624:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205048.669626:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205048.669628:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205048.669630:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205048.669632:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205048.669633:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205048.669635:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205048.669637:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205048.669638:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205048.669639:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205048.669641:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205048.669643:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669647:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669649:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669656:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669657:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205048.669659:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205048.669661:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205048.669663:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205048.669664:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321407517 00000100:00000001:1.0:1788205048.669665:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321408773 00000100:00000001:1.0:1788205048.669667:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321407517 00000100:00000001:1.0:1788205048.669668:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321408773 00000100:00000001:1.0:1788205048.669669:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205048.669670:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669671:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205048.669673:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205048/real 1788205048] req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205048.669677:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205048.669678:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205048.669679:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205048.669680:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205048.669681:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205048.669682:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205048.669683:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205048.669688:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669691:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205048.669692:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669693:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.669694:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205048.669696:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669700:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205048.669702:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669706:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205048.669707:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669708:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.669708:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669709:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205048.669710:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205048.669712:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205048.669713:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205048.669714:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205048.669716:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205048.669720:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504768:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205048.669722:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205048.669723:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669727:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205048.669728:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.669729:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205048.669730:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205048.669731:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205048.669733:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504896:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205048.669734:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205048.669735:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205048.669737:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205048.669738:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205048.669739:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205048.669740:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205048.669742:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205048.669743:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205048.669745:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205048.669746:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543504896, portal 4 00000100:00000001:1.0:1788205048.669747:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205048.669748:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205048.669751:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669755:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205048.669756:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205048.669757:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205048.669758:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543504896, offset 0 00000400:00000200:1.0:1788205048.669760:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205048.669762:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205048.669764:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205048.669765:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205048.669767:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205048.669768:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205048.669770:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205048.669771:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471293:471293:256:4294967295] 192.168.201.118@tcp LPNI seq info [471152:471152:8:4294967295] 00000400:00000200:1.0:1788205048.669774:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205048.669777:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205048.669779:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205048.669781:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205048.669782:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205048.669784:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205048.669786:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205048.669788:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205048.669790:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205048.669792:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205048.669793:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205048.669795:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205048.669796:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205048.669797:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205048.669798:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205048.669800:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669804:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669806:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205073 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669809:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669810:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205048.669811:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205048.669813:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205048.669814:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205048.669815:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321407540 00000100:00000001:1.0:1788205048.669816:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321408773 00000100:00000001:1.0:1788205048.669817:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321407540 00000100:00000001:1.0:1788205048.669819:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321408773 00000100:00000001:1.0:1788205048.669820:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205048.669821:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669822:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205048.669824:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205048/real 1788205048] req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205048.669828:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205048.669828:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205048.669829:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205048.669830:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205048.669831:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205048.669832:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205048.669834:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205048.669839:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669842:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205048.669843:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669843:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.669844:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205048.669846:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669849:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205048.669851:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669854:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205048.669856:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669856:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205048.669857:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669858:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205048.669859:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205048.669860:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205048.669862:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205048.669863:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205048.669865:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205048.669869:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543504896:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205048.669870:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669871:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669873:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543504512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669877:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205048.669878:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205048.669879:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205048.669881:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205048.669881:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205048.669882:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205048.669883:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205048.669884:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205048.669886:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205048.669887:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205048.669887:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205048.669889:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205048.669890:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669891:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669893:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543504640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669897:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205048.669897:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205048.669898:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205048.669899:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205048.669900:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205048.669901:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205048.669902:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205048.669902:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205048.669903:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205048.669904:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205048.669905:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205048.669906:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205048.669907:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669908:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669911:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543504768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669916:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205048.669917:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205048.669919:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205048.669920:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205048.669922:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205048.669923:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205048.669924:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205048.669925:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788205048.669927:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205048.669928:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205048.669930:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205048.669932:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205048.669933:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669934:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205048.669948:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543504896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205073 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205048.669954:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205048.669956:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205048.669957:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205048.669960:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205048.669961:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205048.669962:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205048.669964:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205048.669965:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205048.669967:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205048.669968:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205048.669969:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205048.669971:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205048.669973:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669974:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205048.669977:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205048.669978:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205048.669979:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205048.669980:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205049.500191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205049.500191:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205049.500195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205049.500199:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205049.500200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205049.500201:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205049.500203:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205049.500204:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205049.500205:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205049.500211:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205049.500212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205049.500213:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205049.500216:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788205049.500219:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205049.500220:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205049.500220:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205049.500225:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205049.500225:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205049.500231:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205049.500231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205049.500231:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205049.500236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205049.500236:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205049.500240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205049.500240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205049.500241:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205049.500244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205049.500245:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205049.500252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205049.500255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205049.500259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205049.500260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205049.500320:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205049.500323:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205049.500326:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205050.012213:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321408773, at 4321408832 00000100:00000010:2.0:1788205050.012228:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647540. 00000100:00000040:3.0:1788205050.012317:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321408773, at 4321408832 00000100:00000010:3.0:1788205050.012324:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00000100:00000001:1.0:1788205050.524159:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205050.524166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205050.524166:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205050.524168:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205050.524169:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205050.524175:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205050.524176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205050.524177:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205050.524178:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205050.524180:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205050.524182:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205050.524184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205050.524184:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205050.524187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205050.524188:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205050.524191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205050.524191:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205050.524194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205050.524195:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205050.524197:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205050.524200:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205050.524203:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205050.524204:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205050.524206:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205050.524208:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205050.524216:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205050.524251:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205050.524254:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205050.524256:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205051.548202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205051.548205:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205051.548211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205051.548212:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205051.548212:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205051.548217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205051.548218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205051.548218:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205051.548227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205051.548227:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205051.548227:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205051.548231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205051.548234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205051.548234:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205051.548236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205051.548240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205051.548240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205051.548247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205051.548248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205051.548249:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205051.548251:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205051.548252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205051.548255:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205051.548259:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205051.548259:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205051.548264:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205051.548337:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205051.548342:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205051.548345:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205052.572208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205052.572209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205052.572214:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205052.572219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205052.572220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205052.572222:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205052.572225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205052.572225:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205052.572233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205052.572238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205052.572239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205052.572239:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205052.572247:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205052.572248:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205052.572250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205052.572254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205052.572257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205052.572258:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205052.572260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205052.572264:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205052.572266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205052.572267:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205052.572270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205052.572272:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205052.572278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205052.572279:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205052.572310:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205052.572315:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205052.572319:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205053.596264:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205053.596276:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205053.596279:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205053.596286:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205053.596292:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205053.596296:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205053.596300:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205053.596304:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205053.596307:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596508:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205053.596514:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596522:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205053.596526:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205053.596528:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205053.596533:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596536:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205053.596539:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596541:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596544:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596548:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205053.596550:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205053.596552:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205053.596553:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596555:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205053.596556:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596558:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596560:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596564:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205053.596566:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205053.596568:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205053.596569:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596571:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205053.596572:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596573:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205053.596575:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205053.596927:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205053.596931:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205053.596933:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205053.596936:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205053.596938:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205053.596940:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205053.596943:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205053.596945:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205053.596947:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205053.597188:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205053.597193:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205053.597197:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205053.788325:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205053.788340:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205053.788343:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205053.788348:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205053.788354:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205053.788363:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26740 00000400:00000200:1.0:1788205053.788376:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205053.788385:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205053.788390:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205053.788392:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788395:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.788398:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788401:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205053.788405:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205053.788408:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205053.788411:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205053.788415:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205053.788419:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205053.788422:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205053.788425:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788427:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.788429:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788431:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205053.788434:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205053.788438:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788441:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205053.788443:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205053.788448:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205053.788452:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.788454:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205053.788457:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205053.788469:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788205053.788474:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205053.788478:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205053.788480:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.788484:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205053.788488:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205053.788494:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205053.788504:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb400. 00000100:00000001:1.0:1788205053.788512:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205053.788522:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205053.788532:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205053.788538:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205053.788543:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.788594:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205053.788599:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205053.788602:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205053.788609:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.788617:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205053.788620:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.788625:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205053.788627:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205053.788630:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205053.788633:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543505024:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205053.788637:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205053.788644:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205053.788647:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.788649:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205053.788656:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205053.788657:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205053.788664:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205053.788668:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205053.788679:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205053.788687:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543505024, portal 25 00000100:00000001:3.0:1788205053.788690:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205053.788692:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000040:3.0:1788205053.788698:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.788705:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205053.788707:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205053.788710:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205053.788713:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543505024, offset 0 00000400:00000200:3.0:1788205053.788721:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205053.788732:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.788739:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205053.788745:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205053.788750:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205053.788754:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205053.788757:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205053.788762:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471294:471294:256:4294967295] 192.168.201.118@tcp LPNI seq info [471153:471153:8:4294967295] 00000400:00000200:3.0:1788205053.788766:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.788776:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205053.788782:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.788788:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205053.788796:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205053.788800:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788205053.788806:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205053.788812:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788205053.788817:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.788820:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205053.788828:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205053.788833:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205053.788839:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205053.788841:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205053.788864:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205053.788869:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205053.788893:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205053.788897:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205053.788901:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205053.788902:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205053.788904:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205053.788907:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205053.788910:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26740 00000400:00000200:1.0:1788205053.788914:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205053.788917:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205053.788919:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205053.788922:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788924:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.788926:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788928:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205053.788931:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205053.788932:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205053.788934:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205053.788936:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205053.788938:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205053.788941:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205053.788944:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788946:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.788947:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788949:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205053.788951:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205053.788954:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205053.788956:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205053.788958:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205053.788965:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205053.788967:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.788969:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205053.788974:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205053.788977:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.788978:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205053.788982:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205053.788984:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205053.788986:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205053.788988:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.788989:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205053.788991:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205053.788992:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205053.788996:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba800. 00000100:00000001:1.0:1788205053.788999:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205053.789004:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205053.789016:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205053.789018:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205053.789022:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789050:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.789057:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789064:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.789065:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205053.789075:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205053.789081:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205053.789084:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321413893 00000100:00000010:3.0:1788205053.789110:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcf00. 00000100:00000001:3.0:1788205053.789119:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205053.789128:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321413894 00000100:00000010:3.0:1788205053.789131:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcea0. 00000100:00000001:3.0:1788205053.789135:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205053.789138:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205053.789140:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.789143:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205053.789148:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205053/real 1788205053] req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205053.789159:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205053.789161:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205053.789163:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205053.789165:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205053.789167:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205053.789169:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205053.789174:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205053.789181:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789187:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205053.789189:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789191:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.789194:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205053.789197:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789205:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205053.789209:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789215:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205053.789218:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789219:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.789220:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789223:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205053.789225:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205053.789227:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205053.789230:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205053.789232:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205053.789235:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205053.789242:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543505024:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205053.789247:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789250:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.789252:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543505024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789258:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205053.789261:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205053.789263:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205053.789267:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205053.789269:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205053.789271:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205053.789273:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205053.789276:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:3.0:1788205053.789281:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205053.789284:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205053.789286:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205053.789294:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205053.789296:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789298:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789307:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205053.789310:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.789312:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205053.789314:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205053.789315:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205053.789319:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789326:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205053.789327:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.789329:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205053.789331:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205053.789333:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205053.789335:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505152:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205053.789338:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205053.789340:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205053.789341:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.789343:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205053.789347:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205053.789349:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205053.789353:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205053.789355:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205053.789359:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205053.789362:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543505152, portal 10 00000100:00000001:3.0:1788205053.789364:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205053.789365:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788205053.789372:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789378:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205053.789380:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205053.789383:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205053.789384:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543505152, offset 0 00000400:00000200:3.0:1788205053.789388:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205053.789392:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.789396:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205053.789398:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205053.789400:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205053.789403:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205053.789405:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205053.789407:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471295:471295:256:4294967295] 192.168.201.118@tcp LPNI seq info [471154:471154:8:4294967295] 00000400:00000200:3.0:1788205053.789410:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.789416:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205053.789420:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.789423:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205053.789427:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205053.789429:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788205053.789433:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205053.789436:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788205053.789438:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.789440:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205053.789443:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205053.789447:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205053.789450:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205053.789451:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205053.789454:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205053.789457:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789462:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.789465:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789471:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.789472:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205053.789476:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205053.789480:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205053.789483:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205053.789484:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321412627 00000100:00000001:3.0:1788205053.789487:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321413893 00000100:00000001:3.0:1788205053.789489:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321412627 00000100:00000001:3.0:1788205053.789491:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321413894 00000100:00000001:3.0:1788205053.789493:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205053.789495:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.789497:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205053.789499:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205053/real 1788205053] req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205053.789506:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205053.789507:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205053.789509:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205053.789511:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205053.789512:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205053.789514:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205053.789517:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205053.789524:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789529:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205053.789530:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789531:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.789532:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205053.789535:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789540:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205053.789544:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789550:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205053.789553:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789554:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.789555:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789557:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205053.789559:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205053.789560:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205053.789563:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205053.789565:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205053.789568:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205053.789576:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505152:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205053.789579:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789581:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.789583:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543505152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.789590:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205053.789591:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205053.789593:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205053.789594:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205053.789596:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205053.789597:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205053.789598:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205053.789600:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:3.0:1788205053.789605:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205053.789606:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205053.789607:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788205053.789613:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205053.789614:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789616:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.789619:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205053.789620:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.789622:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205053.789624:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.789692:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205053.789696:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205053.789701:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205053.789703:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205053.789704:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205053.789707:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205053.789710:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26740 00000400:00000200:1.0:1788205053.789713:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205053.789717:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205053.789720:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205053.789723:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789725:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.789728:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789730:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205053.789732:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205053.789733:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205053.789734:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205053.789737:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205053.789739:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205053.789741:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205053.789744:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789746:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.789748:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789750:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205053.789752:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205053.789755:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789757:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205053.789759:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205053.789763:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205053.789765:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.789766:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205053.789769:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205053.789771:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.789773:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205053.789779:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205053.789781:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205053.789783:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205053.789785:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.789786:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205053.789788:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205053.789790:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205053.789794:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000001:1.0:1788205053.789798:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205053.789804:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205053.789810:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205053.789812:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205053.789815:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205053.789823:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205053.789826:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205053.789829:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205053.789830:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205053.789832:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205053.789835:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205053.789841:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26740 00000400:00000200:1.0:1788205053.789843:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205053.789846:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205053.789848:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205053.789863:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789865:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.789868:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789869:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205053.789872:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205053.789873:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205053.789875:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205053.789877:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205053.789879:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205053.789881:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205053.789884:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789886:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.789887:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789889:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205053.789891:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205053.789892:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789894:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205053.789895:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205053.789898:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205053.789900:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.789902:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205053.789904:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205053.789906:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.789909:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205053.789912:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788205053.789914:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205053.789916:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205053.789918:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.789919:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205053.789921:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205053.789922:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205053.789926:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000001:1.0:1788205053.789928:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205053.789932:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205053.789938:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205053.789940:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205053.789943:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205053.789951:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205053.789954:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205053.789957:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205053.789959:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205053.789960:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205053.789964:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205053.789966:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26740 00000400:00000200:1.0:1788205053.789970:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205053.789978:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205053.789980:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205053.789984:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789985:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.789987:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.789989:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205053.789991:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205053.789992:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205053.789993:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205053.789995:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205053.789998:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205053.790001:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205053.790004:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.790006:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205053.790008:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205053.790010:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205053.790012:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205053.790013:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205053.790015:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205053.790017:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205053.790019:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205053.790020:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.790022:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205053.790024:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205053.790025:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.790026:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205053.790030:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205053.790031:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205053.790033:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205053.790035:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205053.790038:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205053.790039:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205053.790040:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205053.790043:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9400. 00000100:00000001:1.0:1788205053.790048:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205053.790052:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205053.790059:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205053.790060:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205053.790064:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205053.790071:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205053.790077:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26750) 00000100:00000001:3.0:1788205053.790204:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205053.790207:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205053.790208:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205053.790214:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790220:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205053.790222:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.790224:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205053.790227:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205053.790229:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205053.790232:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505280:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205053.790235:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205053.790237:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205053.790239:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.790240:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205053.790244:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205053.790245:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205053.790249:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205053.790251:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205053.790255:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205053.790258:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543505280, portal 10 00000100:00000001:3.0:1788205053.790260:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205053.790261:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788205053.790266:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790272:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205053.790274:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205053.790276:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205053.790278:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543505280, offset 0 00000400:00000200:3.0:1788205053.790283:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205053.790287:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.790291:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205053.790294:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205053.790296:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205053.790298:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205053.790301:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205053.790304:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471296:471296:256:4294967295] 192.168.201.118@tcp LPNI seq info [471155:471155:8:4294967295] 00000400:00000200:3.0:1788205053.790307:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.790313:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205053.790318:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.790321:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205053.790325:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205053.790327:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788205053.790331:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205053.790334:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788205053.790337:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.790339:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205053.790343:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205053.790346:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205053.790349:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205053.790351:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205053.790354:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205053.790358:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790365:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.790368:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790373:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.790374:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205053.790377:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205053.790380:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205053.790383:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205053.790386:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321412629 00000100:00000001:3.0:1788205053.790388:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321413893 00000100:00000001:3.0:1788205053.790390:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321412629 00000100:00000001:3.0:1788205053.790392:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321413894 00000100:00000001:3.0:1788205053.790395:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205053.790396:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.790398:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205053.790401:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205053/real 1788205053] req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205053.790407:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205053.790409:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205053.790411:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205053.790413:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205053.790414:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205053.790416:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205053.790419:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205053.790427:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790431:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205053.790432:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.790434:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.790435:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205053.790438:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790443:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205053.790447:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790452:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205053.790456:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.790457:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.790458:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.790460:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205053.790462:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205053.790464:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205053.790467:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205053.790469:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205053.790472:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205053.790480:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505280:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205053.790483:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205053.790486:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790490:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205053.790492:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.790493:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205053.790495:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205053.790497:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205053.790499:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505408:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205053.790502:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205053.790503:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205053.790504:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.790507:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205053.790510:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205053.790512:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205053.790515:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205053.790517:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205053.790521:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205053.790524:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543505408, portal 4 00000100:00000001:3.0:1788205053.790526:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205053.790527:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000040:3.0:1788205053.790532:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790538:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205053.790540:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205053.790542:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205053.790544:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543505408, offset 0 00000400:00000200:3.0:1788205053.790547:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205053.790550:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.790552:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205053.790555:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205053.790557:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205053.790559:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205053.790562:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205053.790565:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471297:471297:256:4294967295] 192.168.201.118@tcp LPNI seq info [471156:471156:8:4294967295] 00000400:00000200:3.0:1788205053.790569:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.790575:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205053.790579:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.790582:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205053.790586:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205053.790589:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788205053.790592:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205053.790595:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788205053.790599:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.790601:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205053.790603:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205053.790607:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205053.790609:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205053.790611:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205053.790612:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205053.790615:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790628:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.790631:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790638:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.790639:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205053.790642:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205053.790644:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205053.790647:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205053.790649:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321412638 00000100:00000001:3.0:1788205053.790652:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321413893 00000100:00000001:3.0:1788205053.790654:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321412638 00000100:00000001:3.0:1788205053.790656:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321413894 00000100:00000001:3.0:1788205053.790658:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205053.790660:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.790662:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205053.790665:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205053/real 1788205053] req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205053.790672:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205053.790674:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205053.790675:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205053.790677:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205053.790679:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205053.790681:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205053.790683:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205053.790691:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790697:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205053.790698:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.790700:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.790701:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205053.790704:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790709:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205053.790712:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790717:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205053.790719:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.790721:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.790722:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.790723:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205053.790725:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205053.790728:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205053.790731:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205053.790732:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205053.790735:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205053.790743:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505408:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205053.790746:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205053.790749:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790754:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205053.790755:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.790757:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205053.790758:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205053.790761:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205053.790764:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505536:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205053.790767:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205053.790769:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205053.790770:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205053.790772:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205053.790776:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205053.790778:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205053.790780:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205053.790783:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205053.790786:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205053.790789:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543505536, portal 4 00000100:00000001:3.0:1788205053.790791:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205053.790792:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205053.790797:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790803:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205053.790805:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205053.790807:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205053.790809:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543505536, offset 0 00000400:00000200:3.0:1788205053.790812:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205053.790815:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.790818:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205053.790820:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205053.790822:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205053.790824:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205053.790827:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205053.790829:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471298:471298:256:4294967295] 192.168.201.118@tcp LPNI seq info [471157:471157:8:4294967295] 00000400:00000200:3.0:1788205053.790833:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205053.790838:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205053.790927:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.790932:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205053.790937:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205053.790940:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:3.1:1788205053.790944:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205053.790946:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:3.0:1788205053.790950:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205053.790952:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205053.790955:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205053.790959:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205053.790962:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205053.790964:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205053.790967:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205053.790970:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790979:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.790981:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205078 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.790987:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.790988:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205053.790992:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205053.790995:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205053.790998:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205053.791000:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321412661 00000100:00000001:3.0:1788205053.791002:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321413893 00000100:00000001:3.0:1788205053.791004:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321412661 00000100:00000001:3.0:1788205053.791006:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321413894 00000100:00000001:3.0:1788205053.791008:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205053.791009:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.791011:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205053.791015:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205053/real 1788205053] req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205053.791021:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205053.791023:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205053.791025:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205053.791027:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205053.791028:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205053.791030:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205053.791032:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205053.791040:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.791045:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205053.791046:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.791047:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.791048:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205053.791051:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.791057:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205053.791061:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.791065:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205053.791068:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.791070:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205053.791071:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.791072:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205053.791074:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205053.791076:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205053.791079:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205053.791081:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205053.791085:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205053.791119:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505536:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205053.791122:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.791124:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.791127:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543505280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.791132:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205053.791134:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205053.791138:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205053.791140:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205053.791142:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205053.791143:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205053.791145:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205053.791147:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:3.0:1788205053.791151:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205053.791152:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205053.791154:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788205053.791157:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205053.791159:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.791161:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.791164:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543505408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.791169:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205053.791170:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205053.791171:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205053.791173:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205053.791174:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205053.791175:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205053.791178:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205053.791180:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:3.0:1788205053.791181:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205053.791182:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205053.791183:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205053.791187:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205053.791188:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.791190:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205053.791194:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543505536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205078 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205053.791199:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205053.791200:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205053.791201:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205053.791204:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205053.791205:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205053.791206:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205053.791208:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205053.791209:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:3.0:1788205053.791211:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205053.791212:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205053.791213:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205053.791214:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205053.791215:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.791217:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205053.791221:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205053.791222:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205053.791226:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205053.791227:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205054.620275:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205054.620276:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205054.620288:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205054.620289:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205054.620293:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205054.620293:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205054.620303:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205054.620304:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205054.620315:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205054.620315:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205054.620322:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205054.620323:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205054.620335:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205054.620335:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205054.620342:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205054.620342:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205054.620352:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205054.620352:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205054.620449:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205054.620454:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205054.620460:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205054.620745:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205054.620751:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205054.620752:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205054.620756:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205054.620757:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205054.621116:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205054.621120:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205054.621123:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205055.132408:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321413893, at 4321413952 00000100:00000010:2.0:1788205055.132836:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf00. 00000100:00000040:3.0:1788205055.132905:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321413894, at 4321413952 00000100:00000010:3.0:1788205055.132913:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcea0. 00000100:00000001:1.0:1788205055.644221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205055.644228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205055.644229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205055.644232:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205055.644235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205055.644239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205055.644243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205055.644243:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205055.644247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205055.644248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205055.644252:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205055.644254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205055.644256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205055.644256:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205055.644264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205055.644264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205055.644267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205055.644304:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205055.644307:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205055.644309:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205055.644387:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205055.644391:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205055.644392:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205055.644396:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205055.644398:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205055.644400:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205055.644403:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205055.644405:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205055.644407:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205056.668231:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205056.668241:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205056.668244:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205056.668251:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205056.668254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205056.668256:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205056.668260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205056.668262:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205056.668266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205056.668268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205056.668270:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205056.668272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205056.668275:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205056.668277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205056.668280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205056.668284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205056.668285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205056.668347:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205056.668353:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205056.668357:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205056.668432:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205056.668436:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205056.668437:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205056.668448:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205056.668450:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205056.668452:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205056.668455:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205056.668457:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205056.668460:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205057.692236:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205057.692243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205057.692248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205057.692249:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205057.692253:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.692253:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205057.692258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205057.692260:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205057.692260:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205057.692261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.692264:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205057.692267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205057.692269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205057.692269:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205057.692271:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205057.692273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.692273:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205057.692275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205057.692278:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205057.692279:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205057.692281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205057.692283:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205057.692285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205057.692285:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205057.692286:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205057.692292:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205057.692409:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205057.692423:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205057.692427:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205058.716205:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205058.716205:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205058.716211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205058.716214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205058.716214:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205058.716217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205058.716218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.716219:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205058.716225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205058.716226:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205058.716228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205058.716230:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205058.716231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.716231:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205058.716234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205058.716236:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205058.716237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205058.716238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205058.716241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205058.716242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.716243:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205058.716244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.716248:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205058.716250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205058.716253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.716254:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205058.716447:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205058.716451:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205058.716455:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205058.908205:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205058.908220:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205058.908223:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205058.908228:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205058.908233:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205058.908242:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26745 00000400:00000200:3.0:1788205058.908254:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205058.908262:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205058.908266:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205058.908269:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908273:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205058.908275:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908277:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205058.908283:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205058.908285:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205058.908288:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205058.908292:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205058.908294:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205058.908297:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205058.908299:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908302:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205058.908304:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908306:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205058.908309:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205058.908313:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908315:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205058.908318:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205058.908332:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205058.908336:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908338:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205058.908341:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205058.908359:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788205058.908363:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205058.908367:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205058.908369:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908373:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205058.908376:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205058.908381:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205058.908390:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412400. 00000100:00000001:3.0:1788205058.908398:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205058.908409:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205058.908428:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205058.908434:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205058.908438:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205058.908462:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205058.908465:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205058.908468:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205058.908470:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205058.908472:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205058.908475:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205058.908480:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26745 00000100:00000001:1.0:1788205058.908485:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1788205058.908485:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205058.908490:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1788205058.908491:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205058.908494:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788205058.908495:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205058.908502:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205058.908502:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908507:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205058.908513:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205058.908515:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205058.908517:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205058.908519:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205058.908522:0:399478:0:(genops.c:814:class_conn2export()) Process entered 02000000:00000001:1.0:1788205058.908525:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1788205058.908525:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 02000000:00000001:1.0:1788205058.908529:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788205058.908529:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205058.908532:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205058.908536:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205058.908539:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205058.908542:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908544:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205058.908545:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908547:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205058.908550:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205058.908553:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908555:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205058.908557:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205058.908564:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205058.908566:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908568:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205058.908572:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205058.908576:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908578:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205058.908582:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205058.908584:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205058.908587:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205058.908588:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205058.908590:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908593:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1788205058.908596:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543505664:192.168.201.118@tcp:250:kworker.0 02000000:00000001:3.0:1788205058.908596:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205058.908600:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.908604:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1788205058.908606:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:1.0:1788205058.908611:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205058.908611:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.908616:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205058.908619:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205058.908621:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788205058.908626:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205058.908629:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205058.908630:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000040:3.0:1788205058.908632:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205058.908634:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908642:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205058.908646:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205058.908647:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205058.908651:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:1.0:1788205058.908653:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00000001:3.0:1788205058.908653:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205058.908655:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205058.908658:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205058.908660:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26745 00000400:00000010:1.0:1788205058.908663:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000400:00000200:3.0:1788205058.908663:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205058.908666:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205058.908669:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205058.908671:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908673:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1788205058.908675:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543505664, portal 25 00000100:00000040:3.0:1788205058.908675:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908678:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205058.908680:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788205058.908682:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:1.0:1788205058.908683:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000020:00000040:3.0:1788205058.908685:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205058.908688:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205058.908691:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000040:1.0:1788205058.908694:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205058.908695:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205058.908702:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205058.908709:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908711:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205058.908713:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908715:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205058.908717:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205058.908719:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908722:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205058.908724:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205058.908726:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205058.908729:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908730:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205058.908733:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205058.908735:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908737:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205058.908741:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788205058.908743:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205058.908746:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205058.908748:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908749:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205058.908750:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205058.908752:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205058.908755:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:3.0:1788205058.908758:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205058.908778:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205058.908786:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205058.908788:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205058.908791:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205058.908799:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205058.908803:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205058.908805:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788205058.908809:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000040:1.0:1788205058.908813:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908813:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205058.908817:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1788205058.908819:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205058.908821:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1788205058.908826:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543505664, offset 0 00000100:00080000:3.0:1788205058.908826:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26745 00000400:00000200:3.0:1788205058.908832:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.908838:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205058.908839:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205058.908844:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205058.908848:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205058.908853:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.908862:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788205058.908863:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205058.908874:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1788205058.908874:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908879:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205058.908883:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000020:00000001:3.0:1788205058.908884:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205058.908887:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205058.908889:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1788205058.908890:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1788205058.908893:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000400:00000200:1.0:1788205058.908896:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000001:3.0:1788205058.908898:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205058.908902:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:1.0:1788205058.908905:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471299:471299:256:4294967295] 192.168.201.118@tcp LPNI seq info [471158:471158:8:4294967295] 00000100:00000040:3.0:1788205058.908908:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908914:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205058.908915:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205058.908918:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908922:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205058.908924:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205058.908929:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205058.908935:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205058.908936:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:3.0:1788205058.908938:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205058.908943:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1788205058.908947:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205058.908948:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908952:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205058.908957:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000800:00000010:1.0:1788205058.908958:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000008:00000001:3.0:1788205058.908963:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.908965:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205058.908970:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:3.0:1788205058.908972:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000800:00000200:1.0:1788205058.908975:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000020:00000040:3.0:1788205058.908977:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205058.908983:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205058.908986:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788205058.908987:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205058.908990:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1788205058.908992:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 02000000:00000001:3.0:1788205058.908993:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205058.908996:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788205058.909001:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205058.909005:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 02000000:00000010:3.0:1788205058.909006:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788205058.909011:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205058.909013:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1788205058.909018:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205058.909022:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205058.909031:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00000001:3.0:1788205058.909031:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205058.909034:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205058.909036:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000040:3.0:1788205058.909041:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.909044:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205058.909051:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205058.909054:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205058.909060:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205058.909064:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788205058.909065:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:1.0:1788205058.909068:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205058.909069:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205058.909073:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205058.909080:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.909080:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205058.909084:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1788205058.909085:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26745 00000100:00000001:1.0:1788205058.909292:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205058.909297:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205058.909300:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321419014 00000100:00000010:1.0:1788205058.909305:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16479c0. 00000100:00000001:1.0:1788205058.909311:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205058.909317:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321419014 00000100:00000010:1.0:1788205058.909364:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647300. 00000100:00000001:1.0:1788205058.909367:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205058.909370:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205058.909372:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.909375:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205058.909379:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205058/real 1788205058] req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205058.909392:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205058.909394:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205058.909395:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205058.909397:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205058.909399:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205058.909401:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205058.909403:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205058.909409:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.909414:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205058.909415:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.909417:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.909418:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205058.909420:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.909424:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205058.909427:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.909431:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205058.909433:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.909433:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.909434:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.909436:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205058.909437:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205058.909440:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205058.909443:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205058.909445:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205058.909449:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205058.909458:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543505664:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205058.909463:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.909465:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205058.909468:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543505664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.909475:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205058.909478:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205058.909482:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205058.909486:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205058.909488:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205058.909490:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205058.909492:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205058.909495:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:1.0:1788205058.909500:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205058.909502:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205058.909504:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205058.909507:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205058.909509:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.909511:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.909520:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205058.909522:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.909525:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205058.909527:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205058.909528:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205058.909532:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.909593:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205058.909595:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.909597:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205058.909599:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205058.909602:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205058.909605:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505792:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205058.909609:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205058.909612:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205058.909614:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.909616:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205058.909619:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205058.909622:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205058.909626:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205058.909629:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205058.909633:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205058.909637:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543505792, portal 10 00000100:00000001:1.0:1788205058.909639:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205058.909640:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205058.909647:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.909654:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205058.909656:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205058.909659:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205058.909661:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543505792, offset 0 00000400:00000200:1.0:1788205058.909666:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205058.909670:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.909674:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205058.909678:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205058.909681:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205058.909683:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205058.909686:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205058.909689:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471300:471300:256:4294967295] 192.168.201.118@tcp LPNI seq info [471159:471159:8:4294967295] 00000400:00000200:1.0:1788205058.909693:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205058.909846:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205058.909962:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205058.909967:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205058.909975:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205058.909979:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788205058.909980:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205058.909986:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.909992:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.0:1788205058.909993:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000020:00000001:3.0:1788205058.909998:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000010:1.0:1788205058.910001:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000020:00000040:3.0:1788205058.910001:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205058.910005:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205058.910009:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000800:00000200:1.0:1788205058.910010:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:3.0:1788205058.910013:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205058.910016:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205058.910020:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.910021:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205058.910024:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205058.910026:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205058.910029:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205058.910032:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205058.910034:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205058.910037:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205058.910041:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205058.910044:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.910046:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205058.910049:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205058.910052:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.910053:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205058.910058:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788205058.910060:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000800:00000200:1.0:1788205058.910154:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205058.910167:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205058.910171:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205058.910177:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205058.910181:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205058.910189:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205058.910196:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1788205058.910199:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205058.910203:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205058.910206:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1788205058.910207:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 02000000:00000001:3.0:1788205058.910209:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1788205058.910213:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 02000000:00000001:3.0:1788205058.910215:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.910224:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:3.0:1788205058.910226:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:3.0:1788205058.910234:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205058.910236:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205058.910245:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205058.910266:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205058.910271:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.910274:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205058.910279:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205058.910286:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.910318:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.910320:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205058.910328:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205058.910334:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205058.910341:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205058.910345:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321417748 00000100:00000001:1.0:1788205058.910349:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321419014 00000100:00000001:1.0:1788205058.910352:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321417748 00000100:00000001:1.0:1788205058.910354:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321419014 00000100:00000001:1.0:1788205058.910356:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205058.910360:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.910362:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205058.910367:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205058/real 1788205058] req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205058.910379:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205058.910381:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205058.910384:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205058.910388:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205058.910390:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205058.910393:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205058.910396:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205058.910405:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.910412:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205058.910413:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.910416:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.910417:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205058.910422:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.910430:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205058.910434:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.910441:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205058.910444:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.910446:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.910447:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.910450:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205058.910453:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205058.910456:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205058.910461:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205058.910463:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205058.910468:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205058.910477:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505792:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205058.910482:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205058.910487:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.910494:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205058.910496:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.910499:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205058.910503:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205058.910506:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205058.910509:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505920:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205058.910513:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205058.910517:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205058.910519:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.910522:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205058.910532:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000001:3.0:1788205058.910532:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.910540:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205058.910546:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205058.910546:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26755) 00000400:00000010:1.0:1788205058.910554:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205058.910560:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205058.910566:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543505920, portal 10 00000100:00000001:1.0:1788205058.910569:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205058.910571:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205058.910578:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.910585:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205058.910588:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205058.910590:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205058.910593:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543505920, offset 0 00000400:00000200:1.0:1788205058.910598:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205058.910605:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.910612:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205058.910617:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205058.910622:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205058.910625:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205058.910628:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205058.910632:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471301:471301:256:4294967295] 192.168.201.118@tcp LPNI seq info [471160:471160:8:4294967295] 00000400:00000200:1.0:1788205058.910636:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.910644:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205058.910649:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205058.910653:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205058.910658:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205058.910661:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205058.910665:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205058.910668:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205058.910686:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205058.910688:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205058.910691:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205058.910694:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205058.910697:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205058.910699:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205058.910701:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205058.910704:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.910712:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205058.910715:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.910721:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.910722:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205058.910725:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205058.910728:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205058.910731:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205058.910733:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321417749 00000100:00000001:1.0:1788205058.910735:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321419014 00000100:00000001:1.0:1788205058.910737:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321417749 00000100:00000001:1.0:1788205058.910740:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321419014 00000100:00000001:1.0:1788205058.910742:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205058.910744:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.910745:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205058.910749:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205058/real 1788205058] req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205058.910756:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205058.910757:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205058.910759:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205058.911166:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205058.911169:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205058.911174:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205058.911179:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205058.911188:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911197:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205058.911200:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.911202:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.911203:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205058.911207:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911213:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205058.911217:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911222:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205058.911225:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.911227:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.911228:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.911230:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205058.911233:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205058.911236:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205058.911239:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205058.911241:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205058.911245:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205058.911252:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543505920:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205058.911349:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205058.911353:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911362:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205058.911363:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.911365:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205058.911367:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205058.911369:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205058.911372:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506048:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205058.911374:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205058.911377:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205058.911378:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.911380:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205058.911383:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205058.911385:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205058.911389:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205058.911391:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205058.911395:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205058.911398:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543506048, portal 4 00000100:00000001:1.0:1788205058.911400:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205058.911401:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205058.911406:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911412:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205058.911414:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205058.911416:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205058.911418:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543506048, offset 0 00000400:00000200:1.0:1788205058.911422:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205058.911426:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.911430:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205058.911433:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205058.911436:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205058.911438:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205058.911440:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205058.911442:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471302:471302:256:4294967295] 192.168.201.118@tcp LPNI seq info [471161:471161:8:4294967295] 00000400:00000200:1.0:1788205058.911446:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.911452:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205058.911456:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205058.911459:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205058.911463:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205058.911465:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205058.911469:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205058.911471:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205058.911475:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205058.911477:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205058.911480:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205058.911483:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205058.911486:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205058.911488:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205058.911490:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205058.911494:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911501:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205058.911504:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911515:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.911516:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205058.911520:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205058.911523:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205058.911526:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205058.911528:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321417759 00000100:00000001:1.0:1788205058.911530:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321419014 00000100:00000001:1.0:1788205058.911532:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321417759 00000100:00000001:1.0:1788205058.911534:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321419014 00000100:00000001:1.0:1788205058.911536:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205058.911538:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.911540:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205058.911543:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205058/real 1788205058] req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205058.911550:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205058.911552:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205058.911553:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205058.911556:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205058.911557:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205058.911560:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205058.911562:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205058.911570:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911575:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205058.911577:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.911578:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.911579:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205058.911583:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911590:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205058.911594:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911600:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205058.911602:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.911603:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.911604:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.911606:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205058.911608:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205058.911610:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205058.911613:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205058.911615:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205058.911618:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205058.911625:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506048:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205058.911628:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.911631:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205058.911633:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543505792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.911640:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205058.911642:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205058.911644:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205058.911648:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205058.911650:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205058.911652:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205058.911654:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205058.911656:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:1.0:1788205058.911660:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205058.911662:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205058.911664:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205058.911667:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205058.911668:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.911669:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205058.912706:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543505920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.912715:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205058.912716:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205058.912718:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205058.912721:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205058.912723:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205058.912724:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205058.912726:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205058.912727:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205058.912730:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205058.912731:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205058.912732:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205058.912734:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205058.912735:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.912737:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205058.912740:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543506048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.912746:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205058.912747:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205058.912748:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205058.912750:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205058.912751:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205058.912752:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205058.912754:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205058.912755:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205058.912757:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205058.912758:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205058.912759:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205058.912771:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205058.912773:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.912775:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.912784:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205058.912785:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.912788:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205058.912790:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205058.912792:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205058.912796:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.912802:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205058.912804:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.912806:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205058.912808:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205058.912810:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205058.912813:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506176:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205058.912816:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205058.912818:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205058.912819:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205058.912821:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205058.912825:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205058.912827:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205058.912831:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205058.912834:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205058.912838:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205058.912842:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543506176, portal 4 00000100:00000001:1.0:1788205058.912844:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205058.912845:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205058.912850:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.912856:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205058.912858:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205058.912870:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205058.912873:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543506176, offset 0 00000400:00000200:1.0:1788205058.912878:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205058.912883:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.912888:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205058.912892:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205058.912895:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205058.912898:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205058.912900:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205058.912903:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471303:471303:256:4294967295] 192.168.201.118@tcp LPNI seq info [471162:471162:8:4294967295] 00000400:00000200:1.0:1788205058.912907:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205058.912914:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205058.912918:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205058.912921:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205058.912926:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205058.912929:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205058.912934:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205058.912937:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205058.912940:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205058.912942:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205058.912945:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205058.912949:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205058.912952:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205058.912955:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205058.912957:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205058.912961:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.912968:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205058.912971:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205083 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.912976:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.912978:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205058.912981:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205058.912984:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205058.912987:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205058.912990:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321417783 00000100:00000001:1.0:1788205058.912992:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321419014 00000100:00000001:1.0:1788205058.912994:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321417783 00000100:00000001:1.0:1788205058.912996:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321419014 00000100:00000001:1.0:1788205058.912998:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205058.913000:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.913002:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205058.913006:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205058/real 1788205058] req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205058.913014:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205058.913015:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205058.913016:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205058.913019:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205058.913020:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205058.913022:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205058.913025:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205058.913032:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.913038:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205058.913039:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.913041:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.913042:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205058.913046:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.913051:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205058.913055:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.913061:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205058.913064:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.913065:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205058.913066:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.913068:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205058.913070:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205058.913073:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205058.913076:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205058.913078:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205058.913081:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205058.913089:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506176:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205058.913109:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.913111:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205058.913114:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543506176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205083 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205058.913120:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205058.913122:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205058.913123:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205058.913126:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205058.913128:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205058.913129:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205058.913139:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205058.913141:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788205058.913143:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205058.913144:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205058.913146:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205058.913148:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205058.913149:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.913151:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205058.913155:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205058.913156:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205058.913158:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205058.913160:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205059.740159:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205059.740166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205059.740173:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205059.740176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205059.740183:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205059.740189:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205059.740193:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205059.740197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205059.740201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205059.740204:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205059.740210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205059.740218:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205059.740222:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205059.740226:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205059.740226:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205059.740230:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205059.740234:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:3.0:1788205059.740235:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205059.740239:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205059.740239:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205059.740242:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205059.740244:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205059.740247:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205059.740249:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205059.740251:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205059.740254:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205059.740261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205059.740264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205059.740278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205059.740279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205059.740283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205059.740284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205059.740326:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205059.740330:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205059.740333:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205060.252248:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321419014, at 4321419072 00000100:00000010:2.0:1788205060.252269:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. 00000100:00000040:3.0:1788205060.252441:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321419014, at 4321419072 00000100:00000010:3.0:1788205060.252448:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00000100:00000001:3.0:1788205060.764248:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205060.764260:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205060.764262:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205060.764269:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205060.764274:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205060.764279:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205060.764283:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205060.764286:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205060.764289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205060.764368:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205060.764373:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205060.764379:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205060.764381:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205060.764385:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205060.764386:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205060.764390:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205060.764391:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205060.764460:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205060.764466:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205060.764470:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205060.765816:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205060.765821:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205060.765823:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205060.765825:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205060.765828:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205060.765830:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205060.765832:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205060.765834:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205060.765836:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205061.788249:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205061.788249:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205061.788258:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205061.788259:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205061.788262:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205061.788278:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205061.788282:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205061.788284:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788205061.788286:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205061.788291:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205061.788291:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205061.788294:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205061.788296:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205061.788299:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205061.788301:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205061.788303:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205061.788307:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205061.788308:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205061.788803:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205061.788807:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205061.788811:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205061.788813:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205061.788816:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205061.788818:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205061.788822:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205061.788823:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205061.788945:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205061.788951:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205061.788956:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205062.812228:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205062.812238:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205062.812240:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205062.812248:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205062.812252:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205062.812256:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205062.812260:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205062.812264:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205062.812267:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205062.812375:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205062.812380:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205062.812385:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205062.812386:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205062.812389:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205062.812391:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205062.812395:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205062.812396:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205062.812439:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205062.812443:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205062.812446:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205062.812484:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205062.812489:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205062.812491:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205062.812494:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205062.812496:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205062.812499:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205062.812501:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205062.812503:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205062.812506:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205063.836229:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205063.836229:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205063.836233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205063.836239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205063.836239:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205063.836243:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205063.836244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205063.836249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205063.836249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205063.836251:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205063.836254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205063.836256:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205063.836257:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205063.836259:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205063.836262:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205063.836264:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205063.836265:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205063.836267:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205063.836272:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205063.836273:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205063.836274:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205063.836279:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205063.836279:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205063.836280:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205063.836282:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205063.836287:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205063.836287:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836288:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836297:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205063.836300:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205063.836301:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205063.836302:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836304:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205063.836306:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836308:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836310:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836314:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205063.836317:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205063.836318:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205063.836320:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836322:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205063.836323:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836325:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205063.836326:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205063.836384:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205063.836390:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205063.836401:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205064.028236:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205064.028247:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205064.028249:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205064.028253:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205064.028257:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205064.028264:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26750 00000400:00000200:2.0:1788205064.028277:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205064.028285:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205064.028288:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205064.028292:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028296:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205064.028298:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028301:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205064.028305:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205064.028308:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205064.028311:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205064.028315:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205064.028318:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205064.028320:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205064.028323:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028324:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205064.028325:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028327:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205064.028329:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205064.028332:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028333:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205064.028336:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205064.028339:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205064.028341:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028343:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205064.028345:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205064.028356:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788205064.028359:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205064.028363:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205064.028364:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028367:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205064.028370:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205064.028375:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205064.028383:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788205064.028390:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205064.028403:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205064.028412:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205064.028416:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205064.028419:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205064.028438:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205064.028440:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205064.028443:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205064.028444:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205064.028445:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205064.028447:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205064.028450:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26750 00000400:00000200:2.0:1788205064.028454:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205064.028457:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205064.028459:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205064.028462:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028464:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205064.028466:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028468:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205064.028470:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205064.028471:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788205064.028473:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205064.028476:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788205064.028478:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788205064.028481:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788205064.028484:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028486:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205064.028487:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788205064.028490:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205064.028493:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205064.028494:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205064.028496:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1788205064.028499:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000040:1.0:1788205064.028506:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205064.028506:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028512:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205064.028516:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205064.028524:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788205064.028524:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205064.028527:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205064.028528:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028533:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205064.028536:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1788205064.028537:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788205064.028540:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788205064.028541:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028544:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.028547:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205064.028548:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788205064.028552:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00100000:1.0:1788205064.028554:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543506304:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788205064.028555:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205064.028558:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.028562:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788205064.028562:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205064.028564:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205064.028566:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.028568:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1788205064.028569:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 02000000:00000001:1.0:1788205064.028571:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.028575:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788205064.028575:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205064.028581:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1788205064.028582:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205064.028587:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028593:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205064.028596:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788205064.028597:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205064.028601:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00000040:2.0:1788205064.028602:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205064.028610:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205064.028611:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00080000:2.0:1788205064.028613:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205064.028616:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788205064.028617:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:1.0:1788205064.028619:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543506304, portal 25 00000100:00080000:2.0:1788205064.028619:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205064.028622:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205064.028624:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205064.028627:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331911296 : -109424377640320 : ffff9c7aa6d74a80) 00000100:00080000:2.0:1788205064.028627:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26750 00000400:00000200:2.0:1788205064.028634:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:1.0:1788205064.028637:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205064.028639:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205064.028644:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205064.028649:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028653:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205064.028656:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028659:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205064.028662:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205064.028664:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788205064.028667:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205064.028671:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000001:1.0:1788205064.028686:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205064.028692:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000020:00000001:2.0:1788205064.028694:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000010:1.0:1788205064.028695:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000020:00000001:2.0:1788205064.028699:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000200:1.0:1788205064.028702:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543506304, offset 0 00000100:00000040:2.0:1788205064.028707:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028710:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205064.028714:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000040:2.0:1788205064.028715:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028719:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205064.028721:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788205064.028723:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028725:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205064.028727:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205064.028729:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788205064.028731:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205064.028734:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028735:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205064.028738:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1788205064.028740:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000008:00000001:2.0:1788205064.028741:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028744:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205064.028749:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000400:00000200:1.0:1788205064.028750:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1788205064.028752:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205064.028755:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205064.028756:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205064.028757:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:2.0:1788205064.028758:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205064.028760:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205064.028762:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205064.028764:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 02000000:00000010:2.0:1788205064.028766:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000400:00000200:1.0:1788205064.028768:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.028771:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205064.028775:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471304:471304:256:4294967295] 192.168.201.118@tcp LPNI seq info [471163:471163:8:4294967295] 00000100:00100000:2.0:1788205064.028779:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205064.028783:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788205064.028790:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205064.028792:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205064.028797:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205064.028801:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:2.0:1788205064.028811:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205064.028814:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788205064.028816:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205064.028819:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000010:1.0:1788205064.028820:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00000001:2.0:1788205064.028820:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205064.028821:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205064.028826:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205064.028828:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26750 00000800:00000200:1.0:1788205064.028833:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:2.0:1788205064.028837:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:1.0:1788205064.028840:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000400:00000200:2.0:1788205064.028841:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205064.028845:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205064.028850:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.1:1788205064.028851:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788205064.028852:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205064.028857:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.1:1788205064.028858:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00000001:2.0:1788205064.028860:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205064.028863:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000100:1.0:1788205064.028866:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000040:2.0:1788205064.028866:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788205064.028867:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000010:1.0:1788205064.028871:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000020:00000040:2.0:1788205064.028871:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788205064.028875:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205064.028877:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788205064.028879:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205064.028880:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205064.028883:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205064.028887:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205064.028889:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788205064.028890:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205064.028894:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:1.0:1788205064.028896:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00080000:2.0:1788205064.028897:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:1.0:1788205064.028901:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:2.0:1788205064.028901:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205064.028905:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205064.028907:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205064.028908:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1788205064.028909:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028912:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:1.0:1788205064.028914:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788205064.028915:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788205064.028920:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028924:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205064.028930:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000100:00000001:1.0:1788205064.028932:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1788205064.028934:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205064.028937:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205064.028938:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.028939:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205064.028940:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205064.028941:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205064.028943:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788205064.028944:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788205064.028948:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205064.028954:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205064.028977:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205064.028982:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.028983:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.028988:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1788205064.028991:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.028997:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205064.029002:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205064.029008:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321424133 00000100:00000010:1.0:1788205064.029014:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647960. 00000100:00000001:2.0:1788205064.029014:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205064.029017:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205064.029021:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205064.029022:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1788205064.029023:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205064.029027:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205064.029032:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321424133 00000100:00000001:2.0:1788205064.029032:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000010:1.0:1788205064.029037:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16472a0. 00000100:00080000:2.0:1788205064.029039:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26750 00000100:00000001:1.0:1788205064.029041:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205064.029046:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1788205064.029049:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205064.029050:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029055:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1788205064.029057:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00100000:1.0:1788205064.029061:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205064/real 1788205064] req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205064.029063:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205064.029071:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.029074:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1788205064.029075:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:2.0:1788205064.029076:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000001:1.0:1788205064.029078:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788205064.029079:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000010:1.0:1788205064.029081:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000020:00000001:2.0:1788205064.029082:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000200:1.0:1788205064.029085:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000020:00000040:2.0:1788205064.029085:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788205064.029086:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:1.0:1788205064.029088:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000020:00000040:2.0:1788205064.029114:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788205064.029117:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788205064.029119:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788205064.029123:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.029125:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205064.029127:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205064.029129:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205064.029131:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:1.0:1788205064.029138:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1788205064.029138:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205064.029143:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205064.029147:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205064.029148:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205064.029154:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1788205064.029157:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205064.029159:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.029164:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205064.029166:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00100000:2.0:1788205064.029168:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205064.029169:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029173:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:2.0:1788205064.029174:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029176:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205064.029180:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205064.029181:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788205064.029187:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788205064.029188:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1788205064.029193:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000040:1.0:1788205064.029194:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205064.029199:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205064.029201:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.029205:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205064.029206:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788205064.029207:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205064.029211:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205064.029211:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029215:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.029215:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1788205064.029216:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:1.0:1788205064.029218:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205064.029221:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788205064.029221:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205064.029223:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205064.029229:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00100000:2.0:1788205064.029229:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205064.029231:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205064.029240:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205064.029240:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205064.029244:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205064.029250:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205064.029261:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205064.029263:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543506304:192.168.201.118@tcp:250:kworker.0 00000100:00000040:2.0:1788205064.029267:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26760) 00000100:00000001:1.0:1788205064.029271:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029274:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.029278:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74a80 x1875055543506304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029283:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205064.029286:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205064.029289:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205064.029292:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205064.029294:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205064.029295:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205064.029297:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205064.029299:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d000. 02000000:00000001:1.0:1788205064.029304:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205064.029306:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205064.029308:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74a80. 00000100:00000001:1.0:1788205064.029311:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205064.029312:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029315:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029324:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205064.029326:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029328:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205064.029330:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205064.029331:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205064.029335:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029342:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205064.029344:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.029346:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205064.029347:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205064.029348:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205064.029351:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506432:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205064.029353:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205064.029354:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205064.029355:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.029356:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205064.029359:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205064.029360:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205064.029363:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205064.029364:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205064.029368:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205064.029370:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543506432, portal 10 00000100:00000001:1.0:1788205064.029371:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205064.029372:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331909504 : -109424377642112 : ffff9c7aa6d74380) 00000100:00000040:1.0:1788205064.029377:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029381:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205064.029383:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205064.029384:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205064.029386:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543506432, offset 0 00000400:00000200:1.0:1788205064.029389:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205064.029392:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205064.029394:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205064.029397:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205064.029398:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205064.029400:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205064.029401:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205064.029403:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471305:471305:256:4294967295] 192.168.201.118@tcp LPNI seq info [471164:471164:8:4294967295] 00000400:00000200:1.0:1788205064.029406:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205064.029411:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205064.029414:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205064.029417:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205064.029421:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205064.029423:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788205064.029428:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205064.029431:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1788205064.029435:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205064.029436:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205064.029440:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205064.029443:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205064.029446:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205064.029448:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205064.029450:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205064.029454:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029461:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.029464:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029471:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029472:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205064.029476:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205064.029480:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205064.029483:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205064.029485:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321422867 00000100:00000001:1.0:1788205064.029488:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321424133 00000100:00000001:1.0:1788205064.029490:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321422867 00000100:00000001:1.0:1788205064.029493:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321424133 00000100:00000001:1.0:1788205064.029495:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205064.029497:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029499:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205064.029502:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205064/real 1788205064] req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205064.029510:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205064.029512:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205064.029514:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205064.029516:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205064.029517:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205064.029519:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205064.029523:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205064.029531:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029537:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205064.029539:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029541:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.029542:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205064.029546:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029553:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205064.029555:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029560:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205064.029562:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029563:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.029564:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029565:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205064.029567:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205064.029568:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205064.029570:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205064.029571:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205064.029573:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205064.029578:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506432:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205064.029581:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205064.029583:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029586:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205064.029587:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.029588:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205064.029589:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205064.029591:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205064.029593:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506560:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205064.029595:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205064.029596:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205064.029597:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.029598:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205064.029601:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205064.029602:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205064.029604:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205064.029606:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205064.029608:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205064.029611:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543506560, portal 10 00000100:00000001:1.0:1788205064.029612:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205064.029613:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913984 : -109424377637632 : ffff9c7aa6d75500) 00000100:00000040:1.0:1788205064.029617:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029621:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205064.029622:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205064.029624:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205064.029625:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543506560, offset 0 00000400:00000200:1.0:1788205064.029628:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205064.029630:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205064.029632:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205064.029634:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205064.029636:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205064.029638:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205064.029639:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205064.029641:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471306:471306:256:4294967295] 192.168.201.118@tcp LPNI seq info [471165:471165:8:4294967295] 00000400:00000200:1.0:1788205064.029644:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205064.029648:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205064.029650:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205064.029652:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205064.029654:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205064.029657:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788205064.029659:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205064.029661:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1788205064.029663:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205064.029665:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205064.029667:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205064.029669:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205064.029671:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205064.029684:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205064.029688:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205064.029691:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029697:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.029699:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029704:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029704:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205064.029707:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205064.029709:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205064.029711:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205064.029713:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321422868 00000100:00000001:1.0:1788205064.029716:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321424133 00000100:00000001:1.0:1788205064.029718:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321422868 00000100:00000001:1.0:1788205064.029720:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321424133 00000100:00000001:1.0:1788205064.029723:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205064.029724:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029727:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205064.029730:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205064/real 1788205064] req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205064.029738:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205064.029739:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205064.029741:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205064.029743:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205064.029745:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205064.029747:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205064.029750:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205064.029759:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029766:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205064.029767:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029769:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.029770:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205064.029774:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029781:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205064.029785:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029791:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205064.029793:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029794:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.029795:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.029797:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205064.029800:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205064.029802:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205064.029806:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205064.029808:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205064.029811:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205064.029817:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506560:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205064.029820:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205064.029823:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029828:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205064.029830:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.029832:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205064.029833:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205064.029835:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205064.029837:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506688:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205064.029840:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205064.029842:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205064.029843:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.029845:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205064.029848:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205064.029850:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205064.029853:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205064.029856:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205064.029860:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205064.029862:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543506688, portal 4 00000100:00000001:1.0:1788205064.029864:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205064.029865:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:1.0:1788205064.029871:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029878:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205064.029880:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205064.029881:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205064.029883:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543506688, offset 0 00000400:00000200:1.0:1788205064.029886:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205064.029889:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205064.029891:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205064.029893:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205064.029895:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205064.029896:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205064.029898:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205064.029900:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471307:471307:256:4294967295] 192.168.201.118@tcp LPNI seq info [471166:471166:8:4294967295] 00000400:00000200:1.0:1788205064.029903:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205064.029906:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205064.029909:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205064.029911:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205064.029914:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205064.029916:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788205064.029918:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205064.029920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1788205064.029923:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205064.029924:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205064.029926:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205064.029928:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205064.029930:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205064.029931:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205064.029933:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205064.029935:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029942:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.029944:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.029950:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029952:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205064.029955:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205064.029958:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205064.029961:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205064.029963:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321422877 00000100:00000001:1.0:1788205064.029966:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321424133 00000100:00000001:1.0:1788205064.029968:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321422877 00000100:00000001:1.0:1788205064.029970:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321424133 00000100:00000001:1.0:1788205064.029973:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205064.029974:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.029976:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205064.029980:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205064/real 1788205064] req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205064.029988:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205064.029989:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205064.029991:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205064.029992:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205064.029993:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205064.029995:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205064.029997:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205064.030002:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030006:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205064.030007:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030008:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.030008:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205064.030010:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030014:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205064.030016:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030020:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205064.030022:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030023:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.030023:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030024:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205064.030025:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205064.030027:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205064.030028:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205064.030030:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205064.030032:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205064.030038:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506688:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205064.030040:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205064.030042:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030045:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205064.030046:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.030047:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205064.030048:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205064.030049:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205064.030051:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506816:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205064.030052:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205064.030053:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205064.030054:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205064.030055:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205064.030057:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205064.030058:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205064.030060:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205064.030062:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205064.030064:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205064.030066:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543506816, portal 4 00000100:00000001:1.0:1788205064.030067:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205064.030068:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:1.0:1788205064.030072:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030076:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205064.030077:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205064.030079:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205064.030080:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543506816, offset 0 00000400:00000200:1.0:1788205064.030083:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205064.030085:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205064.030087:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205064.030108:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205064.030112:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205064.030114:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205064.030116:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205064.030120:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471308:471308:256:4294967295] 192.168.201.118@tcp LPNI seq info [471167:471167:8:4294967295] 00000400:00000200:1.0:1788205064.030124:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205064.030130:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205064.030134:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205064.030138:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205064.030142:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205064.030144:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788205064.030147:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205064.030149:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1788205064.030151:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205064.030153:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205064.030155:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205064.030157:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205064.030160:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205064.030161:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205064.030162:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205064.030165:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030170:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.030172:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205089 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030175:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.030176:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205064.030179:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205064.030181:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205064.030183:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205064.030184:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321422901 00000100:00000001:1.0:1788205064.030186:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321424133 00000100:00000001:1.0:1788205064.030188:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321422901 00000100:00000001:1.0:1788205064.030189:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321424133 00000100:00000001:1.0:1788205064.030190:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205064.030191:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.030192:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205064.030194:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205064/real 1788205064] req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205064.030199:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205064.030200:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205064.030201:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205064.030203:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205064.030204:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205064.030205:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205064.030207:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205064.030211:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030214:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205064.030215:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030216:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.030217:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205064.030218:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030222:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205064.030224:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030228:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205064.030230:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030231:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205064.030232:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030234:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205064.030235:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205064.030239:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205064.030242:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205064.030244:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205064.030247:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205064.030255:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543506816:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205064.030259:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030261:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.030264:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74380 x1875055543506432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030271:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205064.030272:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205064.030274:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205064.030277:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205064.030279:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205064.030280:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205064.030282:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205064.030284:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e800. 02000000:00000001:1.0:1788205064.030287:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205064.030288:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205064.030289:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74380. 00000100:00000001:1.0:1788205064.030291:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205064.030292:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030293:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.030296:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75500 x1875055543506560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030306:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205064.030307:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205064.030309:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205064.030311:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205064.030312:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205064.030313:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205064.030315:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205064.030316:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:1.0:1788205064.030318:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205064.030319:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205064.030321:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75500. 00000100:00000001:1.0:1788205064.030323:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205064.030324:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030326:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.030329:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77b80 x1875055543506688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030336:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205064.030337:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205064.030338:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205064.030340:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205064.030341:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205064.030342:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205064.030345:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205064.030346:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:1.0:1788205064.030348:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205064.030349:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205064.030349:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77b80. 00000100:00000001:1.0:1788205064.030351:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205064.030352:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030354:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205064.030356:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543506816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205089 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205064.030360:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205064.030361:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205064.030362:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205064.030364:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205064.030365:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205064.030366:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205064.030367:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205064.030368:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788205064.030369:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205064.030370:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205064.030370:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788205064.030371:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205064.030372:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030373:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205064.030376:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205064.030377:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205064.030378:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205064.030379:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205064.860237:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205064.860237:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205064.860249:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205064.860250:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205064.860254:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205064.860255:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205064.860264:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205064.860265:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205064.860271:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205064.860271:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205064.860278:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205064.860279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205064.860285:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205064.860286:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205064.860290:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205064.860291:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205064.860296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205064.860296:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205064.860299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205064.860305:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205064.860312:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205064.860314:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205064.860318:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205064.860319:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205064.860324:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205064.860325:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205064.860643:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205064.860648:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205064.860652:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205065.372233:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321424133, at 4321424192 00000100:00000040:3.0:1788205065.372234:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321424133, at 4321424192 00000100:00000010:2.0:1788205065.372254:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647960. 00000100:00000010:3.0:1788205065.372257:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00010000:00000001:0.0:1788205065.884263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205065.884272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205065.884281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205065.884282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205065.884285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205065.884286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205065.884290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205065.884292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205065.884424:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205065.884431:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205065.884433:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205065.884438:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205065.884443:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205065.884446:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205065.884450:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205065.884454:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205065.884457:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205065.885485:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205065.885492:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205065.885496:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205065.885509:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205065.885514:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205065.885516:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205065.885519:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205065.885521:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205065.885523:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205065.885526:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205065.885529:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205065.885531:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205066.908227:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205066.908233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205066.908236:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205066.908237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205066.908239:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205066.908244:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205066.908246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205066.908247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205066.908250:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205066.908251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205066.908252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205066.908254:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205066.908256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205066.908256:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205066.908257:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205066.908258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205066.908262:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205066.908263:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205066.908265:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205066.908266:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205066.908268:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205066.908272:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205066.908274:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205066.908276:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205066.908278:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205066.908279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205066.908335:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205066.908340:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205066.908343:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205067.932238:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205067.932239:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205067.932249:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205067.932250:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205067.932253:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.932253:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205067.932263:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205067.932263:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205067.932270:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205067.932271:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205067.932276:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205067.932276:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205067.932283:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205067.932283:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205067.932287:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205067.932288:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205067.932295:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205067.932295:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205067.932559:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205067.932563:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205067.932569:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205067.932570:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205067.932574:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205067.932575:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205067.932579:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205067.932580:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205067.932674:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205067.932678:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205067.932682:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205068.956242:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205068.956243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205068.956254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205068.956255:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205068.956259:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205068.956263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205068.956266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205068.956268:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205068.956270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205068.956275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205068.956276:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205068.956280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205068.956282:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205068.956284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205068.956289:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205068.956294:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205068.956298:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205068.956360:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205068.956364:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205068.956368:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205068.956673:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205068.956679:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205068.956680:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205068.956692:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205068.956695:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205068.956700:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205068.956704:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205068.956706:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205068.956708:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205069.148266:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205069.148280:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205069.148282:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205069.148287:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205069.148292:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205069.148301:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26755 00000400:00000200:0.0:1788205069.148313:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205069.148322:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205069.148325:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205069.148329:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148333:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205069.148336:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148338:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205069.148343:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205069.148346:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205069.148349:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205069.148353:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205069.148355:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205069.148358:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205069.148360:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148362:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205069.148363:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148365:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205069.148368:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205069.148372:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148374:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205069.148377:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205069.148382:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205069.148385:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.148387:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205069.148390:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205069.148402:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788205069.148406:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205069.148411:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205069.148413:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.148416:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205069.148420:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205069.148424:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205069.148447:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788205069.148455:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205069.148470:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205069.148482:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205069.148488:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205069.148493:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205069.148518:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205069.148522:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205069.148525:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205069.148527:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205069.148528:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205069.148531:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205069.148534:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26755 00000400:00000200:0.0:1788205069.148538:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205069.148541:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205069.148544:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205069.148547:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148549:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205069.148551:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148553:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205069.148555:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205069.148557:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205069.148565:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205069.148568:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205069.148572:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205069.148572:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788205069.148576:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205069.148579:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1788205069.148583:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148600:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205069.148602:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148604:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205069.148606:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788205069.148609:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205069.148610:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205069.148616:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205069.148619:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205069.148619:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205069.148629:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205069.148629:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205069.148632:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205069.148635:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.148637:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788205069.148639:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205069.148640:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205069.148643:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788205069.148646:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205069.148647:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543506944:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205069.148652:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1788205069.148654:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.148658:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.148658:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205069.148661:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205069.148664:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 02000000:00000001:1.0:1788205069.148664:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1788205069.148669:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205069.148672:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000010:1.0:1788205069.148672:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:0.0:1788205069.148674:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.148675:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.148679:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205069.148682:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1788205069.148683:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:0.0:1788205069.148686:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205069.148688:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 02000000:00000010:0.0:1788205069.148693:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000400:00000010:1.0:1788205069.148697:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:0.0:1788205069.148699:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205069.148706:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543506944, portal 25 00000100:00100000:0.0:1788205069.148708:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205069.148709:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205069.148712:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904116736 : -109419805434880 : ffff9c7bb75d9c00) 00000100:00000040:1.0:1788205069.148719:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205069.148723:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205069.148729:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.148729:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205069.148734:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205069.148738:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000040:0.0:1788205069.148739:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205069.148742:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543506944, offset 0 00000400:00000200:1.0:1788205069.148751:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148758:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205069.148762:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205069.148762:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205069.148768:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1788205069.148769:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:0.0:1788205069.148770:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205069.148774:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788205069.148775:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788205069.148781:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205069.148782:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00080000:0.0:1788205069.148788:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26755 00000400:00000200:1.0:1788205069.148788:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205069.148793:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205069.148798:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471309:471309:256:4294967295] 192.168.201.118@tcp LPNI seq info [471168:471168:8:4294967295] 00000400:00000200:1.0:1788205069.148801:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205069.148807:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.148813:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000400:00000200:0.0:1788205069.148818:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205069.148823:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1788205069.148824:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000040:0.0:1788205069.148830:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000010:1.0:1788205069.148832:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00000001:0.0:1788205069.148834:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205069.148838:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148840:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205069.148845:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:1.0:1788205069.148846:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:0.0:1788205069.148848:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788205069.148852:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:1.0:1788205069.148852:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000020:00000040:0.0:1788205069.148855:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205069.148860:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788205069.148864:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000800:00000200:1.1:1788205069.148865:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:0.0:1788205069.148871:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.1:1788205069.148872:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000100:00000001:0.0:1788205069.148874:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205069.148879:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148884:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205069.148886:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000800:00000100:1.0:1788205069.148891:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.148896:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00080000:0.0:1788205069.148902:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205069.148905:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205069.148905:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:0.0:1788205069.148910:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205069.148912:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205069.148915:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.148918:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205069.148919:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000008:00100000:0.0:1788205069.148922:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205069.148926:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.148927:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205069.148929:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00000010:0.0:1788205069.148932:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000400:00000010:1.0:1788205069.148934:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000020:00000040:0.0:1788205069.148936:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205069.148941:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205069.148942:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1788205069.148943:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.148947:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1788205069.148949:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205069.148950:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205069.148952:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205069.148956:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788205069.148959:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205069.148965:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205069.148984:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205069.148987:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.148987:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205069.148992:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205069.148996:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205069.149012:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205069.149017:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205069.149024:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205069.149028:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205069.149031:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205069.149032:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.149035:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205069.149036:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1788205069.149041:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26755 00000400:00000200:0.0:1788205069.149046:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205069.149047:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205069.149051:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:1.0:1788205069.149055:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205069.149056:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205069.149060:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321429253 00000100:00000040:0.0:1788205069.149062:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000010:1.0:1788205069.149066:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647660. 00000100:00000001:0.0:1788205069.149067:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205069.149072:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.149074:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205069.149076:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:1.0:1788205069.149077:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000020:00000040:0.0:1788205069.149078:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205069.149081:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205069.149083:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00000001:1.0:1788205069.149085:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321429253 00000020:00000001:0.0:1788205069.149086:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000010:1.0:1788205069.149116:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647840. 00000100:00000001:1.0:1788205069.149119:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205069.149122:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788205069.149127:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205069.149128:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.149133:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1788205069.149136:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.149141:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788205069.149142:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205069/real 1788205069] req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205069.149146:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.149151:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205069.149155:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788205069.149159:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000001:1.0:1788205069.149159:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205069.149163:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788205069.149164:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205069.149167:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 02000000:00000001:0.0:1788205069.149168:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205069.149170:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205069.149173:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 02000000:00000001:0.0:1788205069.149174:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205069.149175:0:256132:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1788205069.149179:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205069.149181:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205069.149184:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:1.0:1788205069.149190:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788205069.149191:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788205069.149199:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.149202:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788205069.149204:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.149205:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.149209:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.149210:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1788205069.149211:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000100:00100000:1.0:1788205069.149214:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205069.149216:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205069.149222:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205069.149224:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.149228:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205069.149232:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205069.149232:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788205069.149234:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205069.149241:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000040:1.0:1788205069.149241:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205069.149247:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205069.149257:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205069.149257:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205069.149265:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.149268:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.149271:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205069.149274:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205069.149275:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205069.149278:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205069.149280:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205069.149281:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1788205069.149288:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205069.149288:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205069.149290:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205069.149297:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205069.149313:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205069.149315:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543506944:192.168.201.118@tcp:250:kworker.0 00000100:00080000:0.0:1788205069.149318:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205069.149320:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.149324:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788205069.149328:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:1.0:1788205069.149329:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9c00 x1875055543506944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205069.149332:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205069.149336:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205069.149337:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1788205069.149342:0:409030:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1788205069.149342:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205069.149347:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 00000100:00080000:0.0:1788205069.149349:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26755 02000000:00000001:1.0:1788205069.149351:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205069.149354:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000400:00000200:0.0:1788205069.149356:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:1.0:1788205069.149357:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205069.149362:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000400:00000200:0.0:1788205069.149365:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 02000000:00000010:1.0:1788205069.149365:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28000. 00000100:00080000:0.0:1788205069.149371:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788205069.149372:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205069.149376:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:0.0:1788205069.149379:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000010:1.0:1788205069.149380:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9c00. 00000100:00000001:0.0:1788205069.149384:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205069.149386:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000040:0.0:1788205069.149389:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205069.149390:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205069.149396:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205069.149396:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788205069.149402:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205069.149405:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788205069.149407:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205069.149409:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205069.149412:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205069.149415:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788205069.149415:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205069.149419:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205069.149422:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205069.149424:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205069.149428:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205069.149429:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205069.149432:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205069.149433:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1788205069.149437:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:1.0:1788205069.149437:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788205069.149443:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000040:1.0:1788205069.149446:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205069.149447:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205069.149453:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205069.149456:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205069.149461:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788205069.149462:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.149464:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.149466:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205069.149469:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1788205069.149470:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1788205069.149473:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788205069.149475:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.149478:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205069.149480:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205069.149485:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507072:192.168.201.118@tcp:38:kworker.0 00000100:00000010:0.0:1788205069.149487:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000100:00000001:1.0:1788205069.149491:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1788205069.149492:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205069.149495:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205069.149497:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205069.149498:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205069.149501:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.149503:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788205069.149506:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205069.149509:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1788205069.149509:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:0.0:1788205069.149513:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.149514:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205069.149519:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000400:00000200:1.0:1788205069.149520:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205069.149526:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205069.149527:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205069.149535:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00100000:0.0:1788205069.149536:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205069.149539:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543507072, portal 10 00000100:00000001:1.0:1788205069.149545:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205069.149547:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904120320 : -109419805431296 : ffff9c7bb75daa00) 00000100:00000001:0.0:1788205069.149552:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205069.149556:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205069.149561:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205069.149563:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149583:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205069.149598:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205069.149599:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205069.149602:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000040:0.0:1788205069.149607:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26765) 00000100:00000200:1.0:1788205069.149608:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543507072, offset 0 00000400:00000200:1.0:1788205069.149615:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205069.149620:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.149624:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205069.149628:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205069.149631:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205069.149634:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205069.149636:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205069.149640:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471310:471310:256:4294967295] 192.168.201.118@tcp LPNI seq info [471169:471169:8:4294967295] 00000400:00000200:1.0:1788205069.149644:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.149651:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205069.149655:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.149658:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205069.149662:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205069.149664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205069.149669:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205069.149672:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205069.149676:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.149677:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205069.149681:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205069.149684:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205069.149688:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205069.149690:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205069.149693:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205069.149695:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149703:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205069.149706:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149713:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.149714:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205069.149719:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205069.149723:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205069.149727:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205069.149729:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321427987 00000100:00000001:1.0:1788205069.149731:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321429253 00000100:00000001:1.0:1788205069.149734:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321427987 00000100:00000001:1.0:1788205069.149736:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321429253 00000100:00000001:1.0:1788205069.149738:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205069.149740:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.149742:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205069.149746:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205069/real 1788205069] req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205069.149753:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205069.149755:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205069.149756:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205069.149759:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205069.149761:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205069.149762:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205069.149765:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205069.149773:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149780:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205069.149781:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.149783:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.149784:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205069.149787:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149793:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205069.149796:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149801:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205069.149803:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.149804:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.149805:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.149806:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205069.149807:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205069.149809:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205069.149811:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205069.149812:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205069.149814:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205069.149819:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507072:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205069.149822:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205069.149823:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149827:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205069.149828:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.149829:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205069.149831:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205069.149832:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205069.149834:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507200:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205069.149836:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205069.149837:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205069.149838:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.149839:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205069.149842:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205069.149843:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205069.149846:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205069.149848:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205069.149851:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205069.149854:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543507200, portal 10 00000100:00000001:1.0:1788205069.149856:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205069.149857:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904115840 : -109419805435776 : ffff9c7bb75d9880) 00000100:00000040:1.0:1788205069.149862:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149868:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205069.149870:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205069.149873:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205069.149875:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543507200, offset 0 00000400:00000200:1.0:1788205069.149879:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205069.149883:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.149887:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205069.149890:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205069.149893:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205069.149895:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205069.149897:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205069.149901:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471311:471311:256:4294967295] 192.168.201.118@tcp LPNI seq info [471170:471170:8:4294967295] 00000400:00000200:1.0:1788205069.149904:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.149908:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205069.149913:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.149917:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205069.149920:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205069.149923:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205069.149928:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205069.149930:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205069.149934:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.149936:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205069.149938:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205069.149942:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205069.149945:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205069.149947:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205069.149950:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205069.149954:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149962:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205069.149964:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.149971:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.149972:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205069.149976:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205069.149979:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205069.149982:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205069.149984:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321427988 00000100:00000001:1.0:1788205069.149987:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321429253 00000100:00000001:1.0:1788205069.149989:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321427988 00000100:00000001:1.0:1788205069.149991:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321429253 00000100:00000001:1.0:1788205069.149993:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205069.149995:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.149997:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205069.150001:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205069/real 1788205069] req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205069.150009:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205069.150010:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205069.150012:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205069.150015:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205069.150016:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205069.150018:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205069.150022:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205069.150030:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150037:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205069.150039:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150041:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.150042:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205069.150044:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150048:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205069.150051:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150055:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205069.150056:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150057:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.150057:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150058:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205069.150060:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205069.150062:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205069.150063:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205069.150065:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205069.150067:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205069.150071:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507200:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205069.150074:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205069.150076:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150080:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205069.150081:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.150083:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205069.150084:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205069.150086:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205069.150087:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507328:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205069.150108:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205069.150109:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205069.150110:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.150112:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205069.150114:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205069.150116:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205069.150118:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205069.150120:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205069.150122:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205069.150125:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543507328, portal 4 00000100:00000001:1.0:1788205069.150126:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205069.150127:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904124800 : -109419805426816 : ffff9c7bb75dbb80) 00000100:00000040:1.0:1788205069.150131:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150135:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205069.150136:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205069.150138:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205069.150139:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543507328, offset 0 00000400:00000200:1.0:1788205069.150142:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205069.150145:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.150148:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205069.150151:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205069.150153:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205069.150155:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205069.150157:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205069.150160:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471312:471312:256:4294967295] 192.168.201.118@tcp LPNI seq info [471171:471171:8:4294967295] 00000400:00000200:1.0:1788205069.150164:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.150170:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205069.150175:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.150179:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205069.150183:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205069.150185:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205069.150189:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205069.150192:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205069.150195:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.150197:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205069.150199:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205069.150201:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205069.150203:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205069.150205:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205069.150206:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205069.150209:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150215:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205069.150217:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150223:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.150225:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205069.150228:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205069.150231:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205069.150235:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205069.150237:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321427998 00000100:00000001:1.0:1788205069.150239:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321429253 00000100:00000001:1.0:1788205069.150241:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321427998 00000100:00000001:1.0:1788205069.150243:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321429253 00000100:00000001:1.0:1788205069.150245:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205069.150247:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.150249:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205069.150253:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205069/real 1788205069] req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205069.150262:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205069.150264:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205069.150266:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205069.150268:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205069.150270:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205069.150272:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205069.150275:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205069.150284:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150290:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205069.150291:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150293:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.150294:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205069.150297:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150304:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205069.150308:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150315:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205069.150317:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150318:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.150320:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150322:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205069.150324:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205069.150326:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205069.150329:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205069.150332:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205069.150336:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205069.150342:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507328:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205069.150344:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150346:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205069.150348:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75daa00 x1875055543507072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150352:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205069.150353:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205069.150354:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205069.150356:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205069.150357:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205069.150358:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205069.150360:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205069.150361:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a800. 02000000:00000001:1.0:1788205069.150363:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205069.150364:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205069.150365:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75daa00. 00000100:00000001:1.0:1788205069.150367:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205069.150368:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150369:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205069.150371:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9880 x1875055543507200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150375:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205069.150375:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205069.150377:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205069.150378:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205069.150379:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205069.150379:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205069.150380:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205069.150381:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2ac00. 02000000:00000001:1.0:1788205069.150382:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205069.150383:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205069.150384:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9880. 00000100:00000001:1.0:1788205069.150385:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205069.150386:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150387:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205069.150389:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75dbb80 x1875055543507328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150392:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205069.150393:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205069.150394:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205069.150395:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205069.150395:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205069.150396:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205069.150397:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205069.150397:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28c00. 02000000:00000001:1.0:1788205069.150398:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205069.150399:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205069.150400:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dbb80. 00000100:00000001:1.0:1788205069.150401:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205069.150401:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150402:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150406:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205069.150407:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.150409:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205069.150410:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205069.150411:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205069.150414:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150419:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205069.150420:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.150422:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205069.150424:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205069.150425:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205069.150428:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507456:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205069.150431:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205069.150433:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205069.150434:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205069.150436:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205069.150439:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205069.150441:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205069.150444:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205069.150447:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205069.150450:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205069.150453:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543507456, portal 4 00000100:00000001:1.0:1788205069.150455:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205069.150457:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904119424 : -109419805432192 : ffff9c7bb75da680) 00000100:00000040:1.0:1788205069.150462:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150466:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205069.150468:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205069.150470:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205069.150471:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543507456, offset 0 00000400:00000200:1.0:1788205069.150473:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205069.150476:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.150479:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205069.150480:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205069.150482:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205069.150484:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205069.150485:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205069.150487:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471313:471313:256:4294967295] 192.168.201.118@tcp LPNI seq info [471172:471172:8:4294967295] 00000400:00000200:1.0:1788205069.150490:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205069.150494:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205069.150497:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.150499:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205069.150501:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205069.150503:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205069.150506:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205069.150508:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205069.150510:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205069.150512:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205069.150514:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205069.150516:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205069.150518:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205069.150519:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205069.150520:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205069.150522:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150527:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205069.150529:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205094 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150533:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.150534:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205069.150536:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205069.150538:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205069.150539:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205069.150541:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321428021 00000100:00000001:1.0:1788205069.150542:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321429253 00000100:00000001:1.0:1788205069.150544:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321428021 00000100:00000001:1.0:1788205069.150545:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321429253 00000100:00000001:1.0:1788205069.150546:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205069.150547:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.150549:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205069.150552:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205069/real 1788205069] req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205069.150559:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205069.150561:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205069.150562:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205069.150565:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205069.150566:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205069.150568:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205069.150571:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205069.150578:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150595:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205069.150597:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150599:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.150600:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205069.150604:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150611:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205069.150613:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150617:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205069.150619:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150620:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205069.150621:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150622:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205069.150623:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205069.150624:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205069.150626:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205069.150627:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205069.150629:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205069.150633:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507456:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205069.150635:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150637:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205069.150639:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da680 x1875055543507456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205094 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205069.150643:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205069.150644:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205069.150646:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205069.150647:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205069.150648:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205069.150648:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205069.150649:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205069.150651:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b400. 02000000:00000001:1.0:1788205069.150652:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205069.150653:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205069.150654:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da680. 00000100:00000001:1.0:1788205069.150656:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205069.150656:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150658:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205069.150660:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205069.150661:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.150662:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205069.150663:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.980247:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205069.980247:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205069.980258:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205069.980259:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205069.980261:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205069.980263:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205069.980266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205069.980271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205069.980271:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205069.980276:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205069.980280:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205069.980281:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205069.980281:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205069.980287:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205069.980289:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205069.980290:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205069.980293:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205069.980299:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205069.980302:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205069.980304:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205069.980305:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205069.980309:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205069.980313:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205069.980320:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205069.980325:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205069.980329:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205069.980335:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205069.980337:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205069.980342:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205069.980344:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205069.980347:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205069.980349:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205069.980400:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205069.980404:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205069.980407:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205070.492213:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321429253, at 4321429312 00000100:00000040:2.0:1788205070.492218:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321429253, at 4321429312 00000100:00000010:3.0:1788205070.492238:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00000100:00000010:2.0:1788205070.492244:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647660. 00000100:00000001:2.0:1788205071.004213:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205071.004214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205071.004214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205071.004221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205071.004223:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205071.004224:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205071.004227:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205071.004229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205071.004229:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205071.004231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205071.004233:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205071.004239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205071.004239:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788205071.004239:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205071.004243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205071.004244:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205071.004247:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205071.004248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205071.004249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205071.004252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205071.004253:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205071.004254:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205071.004257:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205071.004260:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205071.004265:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205071.004268:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205071.004322:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205071.004327:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205071.004330:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205072.028178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205072.028186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205072.028186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205072.028188:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205072.028190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205072.028194:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205072.028195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205072.028197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205072.028198:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205072.028202:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205072.028205:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205072.028207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205072.028207:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205072.028211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205072.028212:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205072.028221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205072.028223:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205072.028224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205072.028233:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205072.028236:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205072.028242:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205072.028245:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205072.028249:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205072.028252:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205072.028255:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205072.028264:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205072.028276:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205072.028281:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205072.028286:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205073.052194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205073.052197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205073.052202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205073.052203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205073.052204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205073.052211:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205073.052214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205073.052215:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205073.052217:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205073.052218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205073.052220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205073.052228:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205073.052231:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205073.052232:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205073.052233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205073.052236:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205073.052237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205073.052238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205073.052248:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205073.052250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205073.052256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205073.052256:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205073.052261:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205073.052266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205073.052270:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205073.052273:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205073.052320:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205073.052324:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205073.052327:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205074.076178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205074.076178:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205074.076178:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205074.076187:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205074.076187:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205074.076188:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205074.076192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205074.076192:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205074.076200:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205074.076200:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205074.076201:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205074.076206:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205074.076206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205074.076207:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205074.076210:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.076212:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205074.076213:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205074.076215:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788205074.076218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205074.076219:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205074.076221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205074.076223:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.076226:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076227:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205074.076227:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076236:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076240:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205074.076247:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205074.076248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205074.076249:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076251:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205074.076252:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076253:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205074.076258:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205074.076259:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205074.076260:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205074.076261:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076262:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205074.076263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205074.076320:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205074.076323:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205074.076326:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205074.268225:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205074.268235:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205074.268238:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205074.268242:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205074.268246:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205074.268252:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26760 00000400:00000200:0.0:1788205074.268262:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205074.268269:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205074.268272:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205074.268275:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268278:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205074.268280:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268283:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205074.268286:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205074.268288:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205074.268291:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205074.268294:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205074.268297:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205074.268299:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205074.268302:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268304:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205074.268316:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268321:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205074.268324:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205074.268327:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268330:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205074.268333:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205074.268336:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205074.268339:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.268341:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205074.268343:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205074.268352:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788205074.268355:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205074.268359:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205074.268361:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.268364:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205074.268367:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205074.268370:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205074.268378:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788205074.268384:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205074.268392:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205074.268401:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205074.268405:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205074.268410:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205074.268435:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205074.268438:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205074.268441:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205074.268442:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205074.268444:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205074.268447:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205074.268450:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26760 00000400:00000200:0.0:1788205074.268453:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205074.268457:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205074.268460:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205074.268464:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268466:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205074.268469:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205074.268471:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205074.268472:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205074.268476:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788205074.268477:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205074.268481:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:1.0:1788205074.268481:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788205074.268485:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205074.268489:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000040:1.0:1788205074.268490:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788205074.268494:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205074.268513:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788205074.268514:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205074.268516:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.268521:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1788205074.268522:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205074.268524:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205074.268526:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788205074.268527:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205074.268531:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00100000:1.0:1788205074.268533:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543507584:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788205074.268535:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205074.268538:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:0.0:1788205074.268540:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:1.0:1788205074.268543:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788205074.268547:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:1.0:1788205074.268548:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.268552:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.268553:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788205074.268557:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1788205074.268559:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205074.268561:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205074.268563:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205074.268568:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:0.0:1788205074.268569:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205074.268572:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00000001:0.0:1788205074.268573:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205074.268579:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1788205074.268579:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000008:00000001:0.0:1788205074.268585:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.268588:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205074.268588:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543507584, portal 25 00000100:00000001:1.0:1788205074.268593:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000010:0.0:1788205074.268596:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000100:00000001:1.0:1788205074.268596:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904119424 : -109419805432192 : ffff9c7bb75da680) 00000020:00000040:0.0:1788205074.268601:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000040:1.0:1788205074.268602:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205074.268606:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205074.268610:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.268611:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788205074.268615:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1788205074.268615:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000001:0.0:1788205074.268617:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1788205074.268619:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 02000000:00000001:0.0:1788205074.268621:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205074.268623:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543507584, offset 0 02000000:00000010:0.0:1788205074.268627:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000400:00000200:1.0:1788205074.268630:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268633:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.268638:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:0.0:1788205074.268643:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205074.268647:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205074.268656:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788205074.268658:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205074.268661:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:0.0:1788205074.268663:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.268666:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000040:0.0:1788205074.268670:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205074.268671:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205074.268678:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471314:471314:256:4294967295] 192.168.201.118@tcp LPNI seq info [471173:471173:8:4294967295] 00000400:00000200:1.0:1788205074.268687:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205074.268692:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205074.268697:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205074.268698:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:0.0:1788205074.268705:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1788205074.268708:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268709:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205074.268713:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000010:1.0:1788205074.268714:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00000001:0.0:1788205074.268717:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.0:1788205074.268722:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788205074.268724:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26760 00000800:00000200:1.0:1788205074.268726:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000400:00000200:0.0:1788205074.268733:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:1.1:1788205074.268734:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205074.268739:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000400:00000200:0.0:1788205074.268742:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000100:1.0:1788205074.268747:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00080000:0.0:1788205074.268748:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000010:1.0:1788205074.268750:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00000040:0.0:1788205074.268754:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205074.268756:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788205074.268758:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205074.268762:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1788205074.268765:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268768:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205074.268769:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205074.268772:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000020:00000001:0.0:1788205074.268773:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205074.268776:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000100:00000001:1.0:1788205074.268777:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1788205074.268779:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:1.0:1788205074.268783:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205074.268784:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205074.268789:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788205074.268795:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788205074.268795:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.268800:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205074.268802:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.268808:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205074.268810:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205074.268812:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205074.268813:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1788205074.268817:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205074.268819:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205074.268820:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788205074.268826:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:1.0:1788205074.268828:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205074.268830:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205074.268833:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205074.268834:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321434373 02000000:00000001:0.0:1788205074.268836:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205074.268841:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205074.268841:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647900. 00000100:00000001:0.0:1788205074.268846:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205074.268848:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205074.268850:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.268851:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788205074.268853:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205074.268858:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000100:00000001:1.0:1788205074.268859:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321434373 00000020:00000040:0.0:1788205074.268861:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205074.268866:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000010:1.0:1788205074.268866:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647a80. 02000000:00000001:0.0:1788205074.268869:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.268872:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205074.268872:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:0.0:1788205074.268875:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205074.268878:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.268878:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1788205074.268882:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:1.0:1788205074.268883:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.268889:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788205074.268890:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205074.268897:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205074/real 1788205074] req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205074.268898:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205074.268927:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205074.268931:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788205074.268945:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205074.268948:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:0.0:1788205074.268949:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788205074.268951:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205074.268956:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205074.268960:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:0.0:1788205074.268966:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.268966:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1788205074.268971:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1788205074.268974:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205074.268975:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205074.268978:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205074.268981:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205074.268987:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1788205074.268992:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205074.268993:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26760 00000400:00000200:0.0:1788205074.269001:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205074.269007:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205074.269009:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1788205074.269011:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205074.269013:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205074.269016:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205074.269018:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205074.269020:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205074.269022:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205074.269024:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.269028:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205074.269031:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00100000:1.0:1788205074.269031:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205074.269034:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205074.269037:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205074.269041:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00000001:1.0:1788205074.269044:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788205074.269045:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788205074.269050:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205074.269050:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205074.269059:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205074.269061:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205074.269063:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205074.269065:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205074.269068:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205074.269069:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.269071:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205074.269073:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205074.269074:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205074.269076:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:0.0:1788205074.269078:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205074.269079:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788205074.269083:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205074.269085:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205074.269087:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205074.269111:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205074.269115:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.269118:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00080000:1.0:1788205074.269118:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000008:00100000:0.0:1788205074.269123:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:1.0:1788205074.269123:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:0.0:1788205074.269129:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205074.269130:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205074.269134:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205074.269140:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000100:00100000:1.0:1788205074.269143:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543507584:192.168.201.118@tcp:250:kworker.0 00000020:00000040:0.0:1788205074.269145:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788205074.269148:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788205074.269149:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205074.269151:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788205074.269152:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205074.269155:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da680 x1875055543507584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205074.269157:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205074.269159:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205074.269162:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.269164:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205074.269167:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1788205074.269168:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 02000000:00000010:1.0:1788205074.269169:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 00000100:00000001:0.0:1788205074.269173:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.269174:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205074.269176:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205074.269178:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205074.269181:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000100:00100000:0.0:1788205074.269183:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1788205074.269184:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29000. 02000000:00000001:1.0:1788205074.269188:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205074.269190:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205074.269193:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da680. 00000100:00000001:1.0:1788205074.269197:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788205074.269198:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205074.269199:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269202:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205074.269203:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205074.269208:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269208:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205074.269211:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.269215:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205074.269217:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205074.269219:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205074.269224:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205074.269224:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205074.269230:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205074.269234:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205074.269237:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205074.269237:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.269242:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788205074.269242:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205074.269246:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788205074.269247:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788205074.269248:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.269254:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205074.269258:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26760 00000100:00100000:1.0:1788205074.269260:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507712:192.168.201.118@tcp:38:kworker.0 00000400:00000200:0.0:1788205074.269265:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205074.269268:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205074.269271:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:0.0:1788205074.269272:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 02000000:00000001:1.0:1788205074.269273:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205074.269277:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788205074.269277:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205074.269281:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000040:0.0:1788205074.269283:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205074.269285:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.269288:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205074.269291:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:0.0:1788205074.269294:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:1.0:1788205074.269295:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00000001:0.0:1788205074.269300:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000010:1.0:1788205074.269300:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000020:00000001:0.0:1788205074.269306:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000200:1.0:1788205074.269307:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543507712, portal 10 00000020:00000040:0.0:1788205074.269308:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000001:1.0:1788205074.269311:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1788205074.269312:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205074.269314:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904124800 : -109419805426816 : ffff9c7bb75dbb80) 00000020:00000040:0.0:1788205074.269316:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205074.269320:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205074.269321:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788205074.269325:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788205074.269333:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205074.269336:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788205074.269337:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205074.269342:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000040:0.0:1788205074.269343:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.269348:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000010:1.0:1788205074.269349:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000020:00000040:0.0:1788205074.269350:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000200:1.0:1788205074.269355:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543507712, offset 0 00000100:00080000:0.0:1788205074.269356:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205074.269361:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205074.269365:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205074.269366:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 02000000:00000001:0.0:1788205074.269369:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205074.269372:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205074.269374:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205074.269375:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205074.269378:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.269380:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205074.269381:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205074.269387:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000400:00000200:1.0:1788205074.269388:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000040:0.0:1788205074.269392:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000400:00000200:1.0:1788205074.269393:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205074.269398:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000001:0.0:1788205074.269399:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1788205074.269401:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1788205074.269403:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.269405:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:0.0:1788205074.269406:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1788205074.269409:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471315:471315:256:4294967295] 192.168.201.118@tcp LPNI seq info [471174:471174:8:4294967295] 02000000:00000001:0.0:1788205074.269410:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205074.269413:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.269418:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000010:0.0:1788205074.269419:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788205074.269424:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.269429:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00100000:0.0:1788205074.269434:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1788205074.269440:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205074.269448:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205074.269457:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788205074.269460:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1788205074.269463:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000100:00000001:0.0:1788205074.269464:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205074.269471:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788205074.269474:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205074.269480:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000100:00000001:0.0:1788205074.269485:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788205074.269489:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205074.269492:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00000040:0.0:1788205074.269493:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26770) 00000400:00000200:1.0:1788205074.269508:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205074.269514:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205074.269518:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205074.269520:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205074.269523:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205074.269527:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269532:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.269534:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269540:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.269541:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205074.269544:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205074.269548:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205074.269551:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205074.269552:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321433107 00000100:00000001:1.0:1788205074.269572:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321434373 00000100:00000001:1.0:1788205074.269575:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321433107 00000100:00000001:1.0:1788205074.269577:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321434373 00000100:00000001:1.0:1788205074.269579:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205074.269581:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.269583:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205074.269587:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205074/real 1788205074] req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205074.269596:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205074.269597:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205074.269600:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205074.269602:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205074.269604:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205074.269606:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205074.269609:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205074.269618:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269624:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205074.269626:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269627:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.269629:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205074.269632:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269636:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205074.269639:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269643:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205074.269644:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269645:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.269646:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269647:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205074.269648:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205074.269650:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205074.269651:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205074.269653:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205074.269655:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205074.269659:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507712:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205074.269662:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205074.269664:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269668:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205074.269669:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.269670:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205074.269671:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205074.269673:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205074.269674:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507840:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205074.269676:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205074.269677:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205074.269678:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.269679:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205074.269681:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205074.269683:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.269685:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205074.269686:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205074.269688:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205074.269690:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543507840, portal 10 00000100:00000001:1.0:1788205074.269692:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205074.269692:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904115840 : -109419805435776 : ffff9c7bb75d9880) 00000100:00000040:1.0:1788205074.269696:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269700:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205074.269701:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205074.269702:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205074.269704:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543507840, offset 0 00000400:00000200:1.0:1788205074.269706:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205074.269708:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205074.269711:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205074.269712:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205074.269714:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205074.269716:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205074.269718:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205074.269720:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471316:471316:256:4294967295] 192.168.201.118@tcp LPNI seq info [471175:471175:8:4294967295] 00000400:00000200:1.0:1788205074.269722:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205074.269726:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205074.269729:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205074.269730:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205074.269733:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205074.269735:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788205074.269737:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205074.269739:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1788205074.269741:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205074.269743:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205074.269744:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205074.269746:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205074.269748:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205074.269750:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205074.269751:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205074.269753:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269758:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.269759:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269763:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.269764:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205074.269766:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205074.269768:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205074.269770:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205074.269771:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321433108 00000100:00000001:1.0:1788205074.269772:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321434373 00000100:00000001:1.0:1788205074.269774:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321433108 00000100:00000001:1.0:1788205074.269775:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321434373 00000100:00000001:1.0:1788205074.269777:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205074.269778:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.269779:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205074.269781:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205074/real 1788205074] req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205074.269785:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205074.269786:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205074.269787:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205074.269788:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205074.269789:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205074.269790:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205074.269791:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205074.269796:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269800:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205074.269800:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269801:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.269802:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205074.269804:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269807:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205074.269809:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269813:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205074.269814:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269815:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.269815:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269817:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205074.269819:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205074.269820:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205074.269822:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205074.269823:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205074.269825:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205074.269832:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507840:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205074.269835:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269837:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.269840:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75dbb80 x1875055543507712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269846:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205074.269848:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205074.269850:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205074.269852:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205074.269853:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205074.269855:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205074.269857:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205074.269859:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:1.0:1788205074.269861:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205074.269862:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205074.269864:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dbb80. 00000100:00000001:1.0:1788205074.269866:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205074.269867:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269869:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.269872:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9880 x1875055543507840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269878:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205074.269879:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205074.269880:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205074.269881:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205074.269882:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205074.269883:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205074.269884:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205074.269885:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29800. 02000000:00000001:1.0:1788205074.269887:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205074.269887:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205074.269888:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9880. 00000100:00000001:1.0:1788205074.269889:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205074.269890:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269891:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.269894:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205074.269895:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.269896:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205074.269897:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205074.269898:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205074.269900:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269904:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205074.269905:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.269906:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205074.269908:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205074.269909:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205074.269910:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507968:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205074.269912:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205074.269913:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205074.269914:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.269915:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205074.269917:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205074.269919:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.269921:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205074.269923:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205074.269925:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205074.269927:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543507968, portal 4 00000100:00000001:1.0:1788205074.269928:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205074.269930:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904120320 : -109419805431296 : ffff9c7bb75daa00) 00000100:00000040:1.0:1788205074.269933:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269937:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205074.269938:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205074.269939:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205074.269941:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543507968, offset 0 00000400:00000200:1.0:1788205074.269943:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205074.269945:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205074.269947:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205074.269949:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205074.269951:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205074.269952:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205074.269953:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205074.269956:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471317:471317:256:4294967295] 192.168.201.118@tcp LPNI seq info [471176:471176:8:4294967295] 00000400:00000200:1.0:1788205074.269958:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205074.269962:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205074.269965:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205074.269967:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205074.269969:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205074.269971:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788205074.269973:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205074.269975:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1788205074.269977:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205074.269979:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205074.269980:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205074.269982:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205074.269984:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205074.269985:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205074.269986:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205074.269988:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269993:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.269994:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.269999:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.270001:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205074.270003:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205074.270006:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205074.270009:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205074.270011:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321433117 00000100:00000001:1.0:1788205074.270013:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321434373 00000100:00000001:1.0:1788205074.270015:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321433117 00000100:00000001:1.0:1788205074.270017:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321434373 00000100:00000001:1.0:1788205074.270018:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205074.270020:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.270022:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205074.270025:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205074/real 1788205074] req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205074.270032:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205074.270033:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205074.270035:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205074.270038:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205074.270039:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205074.270041:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205074.270044:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205074.270052:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270058:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205074.270059:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270061:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.270062:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205074.270066:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270072:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205074.270076:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270082:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205074.270084:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270086:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.270087:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270089:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205074.270105:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205074.270107:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205074.270110:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205074.270111:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205074.270114:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205074.270118:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543507968:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205074.270121:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205074.270123:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270127:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205074.270128:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.270129:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205074.270130:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205074.270131:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205074.270133:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508096:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205074.270135:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205074.270136:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205074.270137:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205074.270138:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205074.270141:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205074.270142:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205074.270144:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205074.270145:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205074.270148:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205074.270150:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543508096, portal 4 00000100:00000001:1.0:1788205074.270151:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205074.270152:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904116736 : -109419805434880 : ffff9c7bb75d9c00) 00000100:00000040:1.0:1788205074.270156:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270160:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205074.270161:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205074.270162:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205074.270164:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543508096, offset 0 00000400:00000200:1.0:1788205074.270166:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205074.270168:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205074.270170:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205074.270172:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205074.270174:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205074.270175:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205074.270177:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205074.270178:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471318:471318:256:4294967295] 192.168.201.118@tcp LPNI seq info [471177:471177:8:4294967295] 00000400:00000200:1.0:1788205074.270182:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205074.270187:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205074.270192:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205074.270195:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205074.270199:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205074.270201:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788205074.270205:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205074.270207:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1788205074.270211:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205074.270213:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205074.270216:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205074.270219:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205074.270221:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205074.270223:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205074.270225:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205074.270228:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270235:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.270238:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205099 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270244:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.270245:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205074.270248:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205074.270251:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205074.270254:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205074.270256:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321433141 00000100:00000001:1.0:1788205074.270258:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321434373 00000100:00000001:1.0:1788205074.270260:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321433141 00000100:00000001:1.0:1788205074.270262:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321434373 00000100:00000001:1.0:1788205074.270264:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205074.270266:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.270268:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205074.270271:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205074/real 1788205074] req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205074.270279:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205074.270280:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205074.270282:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205074.270284:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205074.270286:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205074.270288:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205074.270292:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205074.270300:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270307:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205074.270308:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270310:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.270311:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205074.270314:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270321:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205074.270324:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270331:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205074.270333:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270334:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205074.270335:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270337:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205074.270339:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205074.270341:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205074.270345:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205074.270347:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205074.270350:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205074.270358:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508096:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205074.270361:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270363:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.270366:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75daa00 x1875055543507968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270372:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205074.270374:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205074.270376:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205074.270378:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205074.270379:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205074.270380:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205074.270381:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205074.270382:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a400. 02000000:00000001:1.0:1788205074.270384:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205074.270385:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205074.270386:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75daa00. 00000100:00000001:1.0:1788205074.270387:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205074.270388:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270389:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205074.270391:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9c00 x1875055543508096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205099 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205074.270395:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205074.270396:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205074.270397:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205074.270398:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205074.270398:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205074.270399:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205074.270400:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205074.270401:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2bc00. 02000000:00000001:1.0:1788205074.270402:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205074.270402:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205074.270403:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9c00. 00000100:00000001:1.0:1788205074.270404:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205074.270405:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270405:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205074.270408:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205074.270409:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205074.270410:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205074.270411:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205075.100188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205075.100188:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205075.100195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205075.100195:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205075.100197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205075.100198:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205075.100205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205075.100205:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205075.100211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205075.100211:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205075.100215:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205075.100215:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205075.100216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205075.100218:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205075.100219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205075.100221:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205075.100223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205075.100224:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205075.100311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205075.100315:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205075.100320:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205075.100322:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205075.100325:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205075.100326:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205075.100329:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205075.100330:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205075.100382:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205075.100386:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205075.100389:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205075.612205:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321434373, at 4321434432 00000100:00000040:3.0:1788205075.612205:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321434373, at 4321434432 00000100:00000010:2.0:1788205075.612221:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647900. 00000100:00000010:3.0:1788205075.612227:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00000100:00000001:1.0:1788205076.124195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205076.124195:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205076.124203:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205076.124204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205076.124205:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205076.124207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205076.124212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205076.124212:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205076.124216:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205076.124217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205076.124220:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205076.124221:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205076.124224:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205076.124227:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205076.124230:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205076.124236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205076.124239:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205076.124242:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205076.124258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205076.124263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205076.124267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205076.124269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205076.124271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205076.124272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205076.124275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205076.124276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205076.124349:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205076.124353:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205076.124356:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205077.148185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205077.148185:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205077.148190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205077.148192:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205077.148192:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205077.148196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205077.148196:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205077.148197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205077.148203:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205077.148203:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205077.148204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205077.148208:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205077.148209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205077.148209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205077.148212:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205077.148213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205077.148216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205077.148216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205077.148216:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205077.148220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205077.148221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205077.148221:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205077.148226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205077.148227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205077.148227:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205077.148230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205077.148282:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205077.148285:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205077.148288:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205078.172210:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205078.172210:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205078.172215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205078.172219:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205078.172219:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205078.172220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205078.172224:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205078.172224:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205078.172227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205078.172231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205078.172232:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205078.172232:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205078.172235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205078.172238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205078.172239:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205078.172239:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205078.172242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205078.172245:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205078.172245:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205078.172246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205078.172251:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205078.172252:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205078.172257:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205078.172257:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205078.172264:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205078.172264:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205078.172302:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205078.172307:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205078.172310:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205079.196200:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205079.196202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205079.196209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205079.196209:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205079.196212:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205079.196213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205079.196213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205079.196219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205079.196220:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205079.196221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205079.196224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205079.196226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.196227:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205079.196227:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205079.196230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205079.196232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205079.196233:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.196234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205079.196235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205079.196238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.196240:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205079.196240:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205079.196244:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205079.196244:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205079.196248:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.196249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205079.196303:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205079.196311:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205079.196315:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205079.388221:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205079.388230:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205079.388237:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205079.388241:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205079.388245:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205079.388252:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26765 00000400:00000200:3.0:1788205079.388262:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205079.388269:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205079.388273:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205079.388276:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388279:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205079.388281:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388284:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205079.388287:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205079.388289:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205079.388291:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205079.388293:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205079.388295:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205079.388297:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205079.388299:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388300:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205079.388302:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388304:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205079.388312:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205079.388316:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388319:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205079.388322:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205079.388326:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205079.388329:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.388331:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205079.388333:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205079.388342:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205079.388346:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205079.388349:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205079.388351:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.388353:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205079.388356:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205079.388361:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205079.388378:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788205079.388384:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205079.388392:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205079.388401:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205079.388405:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205079.388438:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205079.388463:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205079.388466:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205079.388469:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205079.388470:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205079.388472:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205079.388474:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205079.388477:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26765 00000400:00000200:3.0:1788205079.388481:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205079.388484:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205079.388486:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205079.388490:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388492:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205079.388495:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388497:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205079.388499:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788205079.388499:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205079.388501:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:1.0:1788205079.388504:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788205079.388504:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205079.388506:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:1.0:1788205079.388507:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788205079.388511:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205079.388514:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205079.388515:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205079.388519:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388521:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205079.388522:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388526:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205079.388530:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205079.388534:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388539:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205079.388543:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205079.388546:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205079.388550:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205079.388550:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205079.388554:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.388555:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205079.388556:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205079.388560:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1788205079.388562:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788205079.388564:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788205079.388567:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205079.388570:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543508224:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205079.388572:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.388576:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:3.0:1788205079.388578:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 02000000:00000001:1.0:1788205079.388580:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1788205079.388582:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205079.388583:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205079.388587:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205079.388589:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.388591:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205079.388594:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788205079.388594:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205079.388597:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205079.388602:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000010:3.0:1788205079.388603:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205079.388605:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.388611:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205079.388614:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205079.388619:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00100000:3.0:1788205079.388620:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1788205079.388627:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205079.388631:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205079.388634:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205079.388636:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543508224, portal 25 00000100:00000001:1.0:1788205079.388640:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1788205079.388640:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.388644:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205079.388654:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205079.388656:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205079.388662:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205079.388665:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788205079.388666:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205079.388669:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1788205079.388670:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205079.388673:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1788205079.388674:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205079.388676:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1788205079.388679:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543508224, offset 0 00000100:00080000:3.0:1788205079.388680:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26765 00000400:00000200:3.0:1788205079.388685:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.388688:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205079.388691:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205079.388693:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205079.388696:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205079.388700:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205079.388700:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205079.388705:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388708:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205079.388710:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:3.0:1788205079.388712:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205079.388715:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:1.0:1788205079.388717:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1788205079.388718:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205079.388722:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000200:1.0:1788205079.388723:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205079.388729:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205079.388732:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000001:3.0:1788205079.388734:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205079.388738:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471319:471319:256:4294967295] 192.168.201.118@tcp LPNI seq info [471178:471178:8:4294967295] 00000020:00000001:3.0:1788205079.388739:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205079.388745:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205079.388748:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388751:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205079.388758:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205079.388761:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788205079.388762:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205079.388767:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000800:00000200:1.0:1788205079.388771:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205079.388771:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388776:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205079.388778:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:1.0:1788205079.388780:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 02000000:00000001:3.0:1788205079.388780:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205079.388785:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.388786:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205079.388787:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205079.388790:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205079.388791:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205079.388792:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205079.388796:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000800:00000200:1.0:1788205079.388798:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000020:00000040:3.0:1788205079.388799:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205079.388802:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205079.388804:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.388806:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205079.388808:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205079.388810:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788205079.388810:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000010:3.0:1788205079.388814:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000800:00000200:1.1:1788205079.388816:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000100:00000001:3.0:1788205079.388818:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205079.388824:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205079.388827:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.388832:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00000001:3.0:1788205079.388836:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205079.388840:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1788205079.388840:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205079.388848:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1788205079.388849:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205079.388853:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205079.388856:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205079.388861:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1788205079.388863:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205079.388866:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205079.388868:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205079.388876:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205079.388880:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205079.388881:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788205079.388882:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1788205079.388885:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205079.388886:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205079.388891:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26765 00000100:00000001:1.0:1788205079.388894:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.388897:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1788205079.388898:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205079.388902:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205079.388903:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205079.388905:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205079.388906:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388909:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205079.388911:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205079.388912:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205079.388915:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321439493 00000100:00000001:3.0:1788205079.388916:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205079.388920:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205079.388921:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205079.388922:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:1.0:1788205079.388923:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647d80. 00000020:00000040:3.0:1788205079.388926:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205079.388928:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205079.388930:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000020:00000001:3.0:1788205079.388930:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205079.388933:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.388935:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205079.388937:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205079.388938:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321439493 00000100:00000001:3.0:1788205079.388938:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000010:1.0:1788205079.388943:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16474e0. 00000020:00000040:3.0:1788205079.388943:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205079.388946:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205079.388948:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205079.388949:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.388951:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205079.388952:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205079.388955:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205079.388956:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205079.388959:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788205079.388959:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.388962:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1788205079.388965:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205079/real 1788205079] req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205079.388972:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00100000:3.0:1788205079.388972:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000400:00000001:1.0:1788205079.388973:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205079.388976:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000008:00000001:3.0:1788205079.388977:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205079.388978:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205079.388982:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205079.388982:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.388987:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:3.0:1788205079.388989:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000100:00000200:1.0:1788205079.388991:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205079.388995:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205079.389000:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205079.389002:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.389005:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1788205079.389008:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205079.389009:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205079.389013:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.389018:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:3.0:1788205079.389019:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:1.0:1788205079.389020:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389024:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.389026:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205079.389026:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205079.389033:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389037:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205079.389041:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205079.389044:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205079.389052:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205079.389056:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389060:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.389062:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205079.389063:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205079.389064:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205079.389067:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205079.389067:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205079.389071:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1788205079.389072:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205079.389075:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205079.389080:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788205079.389085:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205079.389086:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205079.389089:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205079.389111:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205079.389112:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205079.389113:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205079.389115:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205079.389117:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26765 00000400:00000200:3.0:1788205079.389122:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205079.389127:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205079.389129:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788205079.389132:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543508224:192.168.201.118@tcp:250:kworker.0 00000100:00000040:3.0:1788205079.389134:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205079.389139:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205079.389140:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205079.389142:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205079.389145:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205079.389147:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205079.389151:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000040:1.0:1788205079.389152:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543508224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205079.389154:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205079.389158:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205079.389161:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788205079.389167:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1788205079.389167:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 02000000:00000001:1.0:1788205079.389172:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1788205079.389172:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 02000000:00000010:1.0:1788205079.389177:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 00000100:00000040:3.0:1788205079.389178:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205079.389182:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788205079.389182:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1788205079.389185:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000040:3.0:1788205079.389186:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000020:00000040:1.0:1788205079.389189:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000100:00000001:3.0:1788205079.389190:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205079.389193:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000020:00000040:3.0:1788205079.389195:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 02000000:00000010:1.0:1788205079.389197:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 00000100:00080000:3.0:1788205079.389201:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:1.0:1788205079.389202:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205079.389204:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788205079.389206:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205079.389207:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205079.389211:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1788205079.389211:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205079.389214:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389217:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788205079.389217:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205079.389222:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.389224:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205079.389225:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205079.389227:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788205079.389227:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205079.389230:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205079.389233:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1788205079.389233:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.389237:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205079.389241:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205079.389243:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1788205079.389246:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000040:1.0:1788205079.389248:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205079.389251:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205079.389253:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205079.389256:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205079.389258:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205079.389260:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.389261:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205079.389265:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205079.389265:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205079.389270:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205079.389270:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.389274:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788205079.389277:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205079.389279:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205079.389286:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508352:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205079.389287:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205079.389289:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205079.389292:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389302:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788205079.389305:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.389306:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205079.389309:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205079.389309:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26775) 02000000:00000001:1.0:1788205079.389311:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205079.389316:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205079.389319:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205079.389323:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205079.389327:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205079.389331:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205079.389333:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543508352, portal 10 00000100:00000001:1.0:1788205079.389335:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205079.389336:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788205079.389343:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389349:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205079.389351:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205079.389353:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205079.389355:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543508352, offset 0 00000400:00000200:1.0:1788205079.389359:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205079.389363:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.389367:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205079.389377:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205079.389380:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205079.389382:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205079.389385:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205079.389388:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471320:471320:256:4294967295] 192.168.201.118@tcp LPNI seq info [471179:471179:8:4294967295] 00000400:00000200:1.0:1788205079.389391:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.389397:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205079.389401:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.389404:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205079.389422:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205079.389426:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1788205079.389430:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205079.389433:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000100:1.0:1788205079.389436:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.389438:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205079.389441:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205079.389444:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205079.389447:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205079.389448:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205079.389451:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205079.389454:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389461:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205079.389463:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389470:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.389471:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205079.389474:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205079.389478:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205079.389481:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205079.389483:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321438227 00000100:00000001:1.0:1788205079.389486:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321439493 00000100:00000001:1.0:1788205079.389489:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321438227 00000100:00000001:1.0:1788205079.389491:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321439493 00000100:00000001:1.0:1788205079.389493:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205079.389495:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.389498:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205079.389501:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205079/real 1788205079] req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205079.389508:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205079.389510:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205079.389511:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205079.389514:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205079.389515:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205079.389517:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205079.389519:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205079.389526:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389533:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205079.389535:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389536:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.389538:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205079.389541:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389547:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205079.389552:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389558:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205079.389561:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389563:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.389564:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389566:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205079.389568:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205079.389571:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205079.389574:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205079.389576:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205079.389580:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205079.389587:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508352:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205079.389591:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205079.389594:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389599:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205079.389600:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.389601:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205079.389602:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205079.389604:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205079.389605:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508480:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205079.389608:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205079.389609:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205079.389610:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.389611:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205079.389614:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205079.389616:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205079.389619:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205079.389621:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205079.389625:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205079.389627:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543508480, portal 10 00000100:00000001:1.0:1788205079.389629:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205079.389631:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205079.389636:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389641:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205079.389644:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205079.389646:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205079.389648:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543508480, offset 0 00000400:00000200:1.0:1788205079.389652:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205079.389656:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.389660:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205079.389663:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205079.389665:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205079.389667:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205079.389669:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205079.389672:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471321:471321:256:4294967295] 192.168.201.118@tcp LPNI seq info [471180:471180:8:4294967295] 00000400:00000200:1.0:1788205079.389674:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.389679:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205079.389682:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.389684:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205079.389686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205079.389688:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1788205079.389691:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205079.389692:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000100:1.0:1788205079.389696:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.389697:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205079.389700:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205079.389704:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205079.389706:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205079.389708:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205079.389711:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205079.389714:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389727:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205079.389730:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389737:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.389738:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205079.389741:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205079.389743:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205079.389746:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205079.389747:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321438228 00000100:00000001:1.0:1788205079.389749:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321439493 00000100:00000001:1.0:1788205079.389750:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321438228 00000100:00000001:1.0:1788205079.389751:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321439493 00000100:00000001:1.0:1788205079.389753:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205079.389754:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.389755:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205079.389758:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205079/real 1788205079] req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205079.389765:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205079.389766:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205079.389768:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205079.389770:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205079.389772:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205079.389774:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205079.389777:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205079.389784:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389790:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205079.389792:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389793:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.389795:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205079.389798:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389805:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205079.389808:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389815:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205079.389817:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389819:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.389820:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389822:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205079.389824:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205079.389825:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205079.389827:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205079.389829:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205079.389831:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205079.389836:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508480:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205079.389838:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205079.389840:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389844:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205079.389845:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.389847:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205079.389849:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205079.389851:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205079.389854:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508608:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205079.389857:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205079.389859:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205079.389860:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.389862:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205079.389865:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205079.389866:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205079.389869:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205079.389872:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205079.389875:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205079.389878:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543508608, portal 4 00000100:00000001:1.0:1788205079.389880:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205079.389881:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205079.389886:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389893:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205079.389895:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205079.389897:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205079.389899:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543508608, offset 0 00000400:00000200:1.0:1788205079.389902:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205079.389905:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.389907:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205079.389909:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205079.389911:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205079.389912:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205079.389913:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205079.389915:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471322:471322:256:4294967295] 192.168.201.118@tcp LPNI seq info [471181:471181:8:4294967295] 00000400:00000200:1.0:1788205079.389918:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.389922:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205079.389924:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.389926:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205079.389929:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205079.389931:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1788205079.389933:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205079.389935:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000100:1.0:1788205079.389937:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.389939:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205079.389940:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205079.389942:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205079.389944:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205079.389945:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205079.389947:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205079.389949:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389953:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205079.389955:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389959:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.389960:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205079.389962:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205079.389964:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205079.389966:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205079.389967:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321438237 00000100:00000001:1.0:1788205079.389968:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321439493 00000100:00000001:1.0:1788205079.389970:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321438237 00000100:00000001:1.0:1788205079.389971:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321439493 00000100:00000001:1.0:1788205079.389972:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205079.389973:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.389974:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205079.389976:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205079/real 1788205079] req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205079.389981:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205079.389982:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205079.389983:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205079.389984:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205079.389985:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205079.389987:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205079.389988:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205079.389993:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.389997:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205079.389997:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.389998:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.389999:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205079.390001:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390004:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205079.390006:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390010:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205079.390011:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390012:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.390012:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390013:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205079.390015:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205079.390016:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205079.390018:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205079.390019:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205079.390021:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205079.390028:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508608:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205079.390030:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390032:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205079.390033:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543508352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390037:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205079.390037:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205079.390039:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205079.390040:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205079.390041:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205079.390041:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205079.390042:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205079.390043:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205079.390045:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205079.390045:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205079.390046:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205079.390048:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205079.390049:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390050:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205079.390051:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543508480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390055:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205079.390056:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205079.390057:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205079.390058:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205079.390059:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205079.390059:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205079.390060:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205079.390061:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205079.390062:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205079.390063:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205079.390063:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205079.390065:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205079.390066:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390067:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205079.390069:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543508608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390072:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205079.390073:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205079.390073:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205079.390075:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205079.390075:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205079.390076:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205079.390077:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205079.390077:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205079.390078:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205079.390079:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205079.390079:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205079.390080:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205079.390081:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390082:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390084:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205079.390085:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.390086:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205079.390087:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205079.390088:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205079.390104:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390108:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205079.390109:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.390110:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205079.390111:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205079.390112:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205079.390114:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508736:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205079.390116:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205079.390117:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205079.390118:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205079.390119:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205079.390121:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205079.390122:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205079.390124:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205079.390126:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205079.390127:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205079.390129:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543508736, portal 4 00000100:00000001:1.0:1788205079.390130:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205079.390131:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788205079.390135:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390138:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205079.390140:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205079.390141:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205079.390142:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543508736, offset 0 00000400:00000200:1.0:1788205079.390144:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205079.390147:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.390150:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205079.390152:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205079.390155:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205079.390157:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205079.390159:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205079.390161:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471323:471323:256:4294967295] 192.168.201.118@tcp LPNI seq info [471182:471182:8:4294967295] 00000400:00000200:1.0:1788205079.390165:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205079.390171:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205079.390176:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.390179:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205079.390183:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205079.390186:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000200:1.1:1788205079.390189:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205079.390192:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 23 secs later) 00000800:00000100:1.0:1788205079.390195:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205079.390197:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205079.390199:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205079.390201:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205079.390203:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205079.390204:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205079.390205:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205079.390207:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390212:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205079.390214:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205104 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390218:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.390219:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205079.390220:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205079.390222:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205079.390224:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205079.390226:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321438261 00000100:00000001:1.0:1788205079.390227:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321439493 00000100:00000001:1.0:1788205079.390229:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321438261 00000100:00000001:1.0:1788205079.390230:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321439493 00000100:00000001:1.0:1788205079.390231:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205079.390232:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.390233:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205079.390235:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205079/real 1788205079] req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205079.390240:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205079.390241:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205079.390242:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205079.390243:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205079.390244:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205079.390245:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205079.390247:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205079.390252:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390256:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205079.390256:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390257:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.390258:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205079.390260:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390263:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205079.390265:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390268:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205079.390270:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390270:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205079.390271:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390272:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205079.390273:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205079.390274:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205079.390276:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205079.390277:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205079.390280:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205079.390284:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508736:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205079.390286:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390287:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205079.390288:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543508736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205104 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205079.390292:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205079.390293:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205079.390294:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205079.390296:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205079.390296:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205079.390297:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205079.390298:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205079.390300:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205079.390302:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205079.390303:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205079.390304:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205079.390307:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205079.390308:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390310:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205079.390313:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205079.390314:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205079.390315:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205079.390317:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205080.220182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205080.220182:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205080.220184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205080.220189:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205080.220189:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205080.220191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205080.220192:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205080.220192:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205080.220196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205080.220197:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205080.220198:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205080.220202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205080.220203:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205080.220203:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205080.220208:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205080.220209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205080.220209:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205080.220210:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205080.220214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205080.220215:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205080.220215:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205080.220220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205080.220220:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205080.220222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205080.220225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205080.220226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205080.220226:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205080.220232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205080.220234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205080.220238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205080.220238:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205080.220240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205080.220296:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205080.220299:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205080.220301:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205080.732175:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321439493, at 4321439552 00000100:00000040:3.0:1788205080.732175:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321439493, at 4321439552 00000100:00000010:3.0:1788205080.732188:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16474e0. 00000100:00000010:2.0:1788205080.732190:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00000100:00000001:1.0:1788205081.244197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205081.244197:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205081.244201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205081.244205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205081.244206:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205081.244206:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205081.244211:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205081.244212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205081.244212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205081.244216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205081.244219:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205081.244220:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205081.244221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205081.244225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205081.244227:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205081.244228:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205081.244231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205081.244234:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205081.244234:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205081.244235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205081.244242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205081.244242:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205081.244247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205081.244247:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205081.244251:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205081.244252:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205081.244307:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205081.244311:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205081.244314:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205082.268197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205082.268206:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205082.268208:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205082.268213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205082.268217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205082.268220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205082.268221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205082.268223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205082.268227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205082.268227:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205082.268234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205082.268236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205082.268239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205082.268242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205082.268244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205082.268248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205082.268250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205082.268318:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205082.268323:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205082.268326:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205082.268436:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205082.268443:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205082.268445:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205082.268448:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205082.268451:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205082.268452:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205082.268455:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205082.268457:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205082.268459:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205083.292188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205083.292192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205083.292196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205083.292197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205083.292200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205083.292206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205083.292207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205083.292210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205083.292211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205083.292214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205083.292216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205083.292217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205083.292222:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205083.292225:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205083.292228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205083.292232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205083.292234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205083.292297:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205083.292301:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205083.292311:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205083.292313:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205083.292317:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205083.292319:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205083.292322:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205083.292325:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205083.292327:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205083.292330:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205083.292332:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205083.292334:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.316205:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205084.316210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205084.316213:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205084.316214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205084.316216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.316218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205084.316223:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205084.316225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205084.316225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205084.316226:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316230:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205084.316231:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788205084.316232:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205084.316235:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205084.316237:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205084.316238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205084.316240:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205084.316242:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.316244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205084.316246:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205084.316248:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205084.316249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205084.316250:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205084.316252:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205084.316255:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.316256:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205084.316258:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316260:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205084.316268:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205084.316270:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205084.316271:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316282:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205084.316284:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316290:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316293:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316297:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205084.316299:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205084.316301:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205084.316303:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316305:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205084.316307:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316308:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205084.316310:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205084.316377:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205084.316381:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205084.316385:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205084.508210:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205084.508220:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205084.508222:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205084.508225:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205084.508229:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205084.508234:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26770 00000400:00000200:3.0:1788205084.508242:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205084.508249:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205084.508252:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205084.508255:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508257:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205084.508260:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508262:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205084.508266:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205084.508268:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205084.508272:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205084.508276:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205084.508278:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205084.508281:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205084.508284:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508286:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205084.508288:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508290:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205084.508293:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205084.508296:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508299:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205084.508301:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205084.508304:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205084.508306:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.508307:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205084.508309:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205084.508317:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205084.508343:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205084.508346:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205084.508347:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.508349:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205084.508351:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205084.508354:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205084.508361:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412c00. 00000100:00000001:3.0:1788205084.508367:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205084.508373:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205084.508380:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205084.508383:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205084.508387:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205084.508405:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205084.508407:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205084.508410:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205084.508411:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205084.508412:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205084.508414:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205084.508415:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26770 00000400:00000200:3.0:1788205084.508418:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205084.508420:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205084.508422:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205084.508424:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508425:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205084.508427:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508429:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205084.508431:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205084.508433:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205084.508435:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205084.508437:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205084.508439:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205084.508440:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788205084.508442:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205084.508445:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1788205084.508448:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205084.508449:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788205084.508453:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205084.508458:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205084.508459:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508465:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205084.508468:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205084.508471:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508476:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.508479:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788205084.508480:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205084.508483:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.508487:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205084.508487:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205084.508490:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205084.508491:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.508496:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.508496:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205084.508500:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205084.508501:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543508864:192.168.201.118@tcp:250:kworker.0 00000008:00000001:3.0:1788205084.508507:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.508509:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788205084.508512:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.508515:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1788205084.508517:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 02000000:00000001:1.0:1788205084.508519:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205084.508522:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205084.508526:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205084.508527:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.508529:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205084.508531:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788205084.508531:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205084.508535:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205084.508540:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000010:3.0:1788205084.508540:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788205084.508545:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205084.508553:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205084.508557:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205084.508565:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205084.508576:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543508864, portal 25 00000100:00000001:1.0:1788205084.508578:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205084.508590:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788205084.508596:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.508604:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788205084.508606:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205084.508609:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205084.508613:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00100000:3.0:1788205084.508616:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205084.508619:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543508864, offset 0 00000100:00000001:3.0:1788205084.508630:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205084.508633:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205084.508638:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205084.508641:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508651:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205084.508653:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205084.508656:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205084.508661:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1788205084.508663:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788205084.508664:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205084.508665:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205084.508668:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205084.508673:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26770 00000400:00000200:1.0:1788205084.508679:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205084.508681:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.508686:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205084.508687:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205084.508691:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1788205084.508694:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1788205084.508695:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508699:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205084.508703:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000040:3.0:1788205084.508703:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508708:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205084.508710:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205084.508712:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205084.508714:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1788205084.508715:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471324:471324:256:4294967295] 192.168.201.118@tcp LPNI seq info [471183:471183:8:4294967295] 00000020:00000040:3.0:1788205084.508716:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205084.508720:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205084.508723:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205084.508727:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205084.508733:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508735:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205084.508737:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508738:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205084.508742:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205084.508747:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:1.0:1788205084.508748:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788205084.508750:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205084.508753:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.0:1788205084.508761:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205084.508762:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205084.508766:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.508768:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000010:1.0:1788205084.508769:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000008:00100000:3.0:1788205084.508771:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205084.508776:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.508777:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205084.508781:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:3.0:1788205084.508784:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000800:00000200:1.0:1788205084.508786:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000020:00000040:3.0:1788205084.508788:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205084.508794:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205084.508797:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788205084.508798:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205084.508801:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1788205084.508803:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 02000000:00000001:3.0:1788205084.508804:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205084.508807:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788205084.508812:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000010:3.0:1788205084.508813:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412000. 00000800:00000010:1.0:1788205084.508816:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00000001:3.0:1788205084.508819:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205084.508825:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1788205084.508825:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205084.508834:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205084.508836:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788205084.508837:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205084.508842:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205084.508846:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205084.508851:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205084.508851:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205084.508855:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205084.508858:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205084.508859:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1788205084.508859:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205084.508862:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205084.508864:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205084.508867:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26770 00000100:00000200:1.0:1788205084.508869:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205084.508879:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205084.508881:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1788205084.508885:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000040:1.0:1788205084.508889:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205084.508890:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205084.508896:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508901:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205084.508906:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205084.508906:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508910:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205084.508911:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1788205084.508915:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205084.508917:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000100:00000001:1.0:1788205084.508919:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205084.508919:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205084.508923:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205084.508927:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:1.0:1788205084.508928:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:3.0:1788205084.508929:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205084.508934:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321444613 00000100:00000040:3.0:1788205084.508934:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508936:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205084.508939:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000010:1.0:1788205084.508943:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647c00. 00000100:00000001:3.0:1788205084.508947:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205084.508950:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000001:1.0:1788205084.508952:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00080000:3.0:1788205084.508953:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205084.508957:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205084.508958:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205084.508961:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205084.508962:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321444613 02000000:00000001:3.0:1788205084.508962:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.508967:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000010:1.0:1788205084.508969:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647240. 00000008:00100000:3.0:1788205084.508969:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205084.508974:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.508976:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205084.508976:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205084.508979:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205084.508981:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1788205084.508982:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788205084.508988:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205084.508988:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205084.508989:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.508994:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205084.508995:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788205084.508995:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205084.508999:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205084.509003:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00100000:1.0:1788205084.509005:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205084/real 1788205084] req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205084.509006:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205084.509013:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205084.509028:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1788205084.509030:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788205084.509030:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788205084.509035:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1788205084.509035:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788205084.509039:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205084.509044:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205084.509049:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205084.509049:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509054:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1788205084.509055:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205084.509061:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000200:1.0:1788205084.509063:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205084.509063:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205084.509067:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205084.509072:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205084.509077:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26770 00000100:00000200:1.0:1788205084.509082:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205084.509085:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205084.509119:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1788205084.509121:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205084.509123:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205084.509126:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205084.509127:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.509132:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205084.509133:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.509137:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788205084.509141:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205084.509142:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.509147:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205084.509153:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205084.509155:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205084.509159:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205084.509160:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1788205084.509162:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205084.509167:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205084.509169:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205084.509171:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205084.509180:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205084.509185:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205084.509187:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1788205084.509189:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205084.509192:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205084.509194:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205084.509197:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:3.0:1788205084.509199:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205084.509202:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205084.509205:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205084.509207:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205084.509210:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205084.509214:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509215:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205084.509216:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205084.509219:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205084.509220:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:3.0:1788205084.509222:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509225:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205084.509225:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205084.509230:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000100:00080000:1.0:1788205084.509232:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:3.0:1788205084.509233:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205084.509238:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205084.509239:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205084.509241:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1788205084.509242:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 02000000:00000001:3.0:1788205084.509243:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205084.509246:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509248:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000010:3.0:1788205084.509250:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:3.0:1788205084.509254:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205084.509260:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1788205084.509264:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205084.509268:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205084.509272:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205084.509280:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205084.509292:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205084.509297:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543508864:192.168.201.118@tcp:250:kworker.0 00000100:00000040:3.0:1788205084.509297:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26781) 00000100:00000001:1.0:1788205084.509307:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509310:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.509314:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543508864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509333:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205084.509336:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205084.509339:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205084.509342:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205084.509345:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205084.509346:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205084.509348:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205084.509350:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:1.0:1788205084.509354:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205084.509355:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205084.509357:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205084.509361:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205084.509362:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509365:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509371:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205084.509373:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509376:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205084.509377:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205084.509379:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205084.509382:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509389:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205084.509391:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.509393:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205084.509395:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205084.509397:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205084.509400:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508992:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205084.509402:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205084.509405:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205084.509406:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.509408:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205084.509411:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205084.509414:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205084.509417:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205084.509419:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205084.509423:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205084.509426:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543508992, portal 10 00000100:00000001:1.0:1788205084.509427:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205084.509428:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205084.509435:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509441:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205084.509443:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205084.509446:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205084.509448:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543508992, offset 0 00000400:00000200:1.0:1788205084.509452:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205084.509456:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.509460:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205084.509463:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205084.509465:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205084.509468:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205084.509470:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205084.509473:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471325:471325:256:4294967295] 192.168.201.118@tcp LPNI seq info [471184:471184:8:4294967295] 00000400:00000200:1.0:1788205084.509477:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.509483:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205084.509488:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205084.509491:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205084.509495:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205084.509498:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205084.509502:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205084.509505:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205084.509508:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205084.509509:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205084.509513:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205084.509516:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205084.509519:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205084.509521:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205084.509523:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205084.509527:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509535:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.509538:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509544:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509547:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205084.509550:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205084.509554:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205084.509557:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205084.509559:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321443347 00000100:00000001:1.0:1788205084.509561:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321444613 00000100:00000001:1.0:1788205084.509563:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321443347 00000100:00000001:1.0:1788205084.509565:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321444613 00000100:00000001:1.0:1788205084.509567:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205084.509569:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509571:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205084.509574:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205084/real 1788205084] req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205084.509582:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205084.509583:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205084.509585:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205084.509588:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205084.509589:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205084.509591:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205084.509594:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205084.509603:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509609:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205084.509610:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509612:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.509613:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205084.509616:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509622:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205084.509626:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509631:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205084.509634:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509635:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.509636:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509638:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205084.509640:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205084.509642:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205084.509645:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205084.509647:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205084.509650:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205084.509657:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543508992:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205084.509661:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205084.509664:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509676:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205084.509678:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.509680:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205084.509681:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205084.509683:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205084.509686:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509120:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205084.509689:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205084.509691:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205084.509692:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.509694:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205084.509697:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205084.509699:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205084.509702:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205084.509704:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205084.509707:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205084.509709:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543509120, portal 10 00000100:00000001:1.0:1788205084.509711:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205084.509712:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205084.509717:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509723:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205084.509725:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205084.509727:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205084.509729:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543509120, offset 0 00000400:00000200:1.0:1788205084.509732:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205084.509736:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.509739:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205084.509741:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205084.509744:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205084.509746:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205084.509748:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205084.509751:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471326:471326:256:4294967295] 192.168.201.118@tcp LPNI seq info [471185:471185:8:4294967295] 00000400:00000200:1.0:1788205084.509755:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.509760:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205084.509765:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205084.509767:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205084.509771:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205084.509774:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205084.509778:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205084.509780:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205084.509784:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205084.509786:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205084.509789:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205084.509792:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205084.509795:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205084.509796:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205084.509798:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205084.509802:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509818:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.509824:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509830:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509831:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205084.509834:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205084.509836:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205084.509839:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205084.509841:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321443348 00000100:00000001:1.0:1788205084.509844:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321444613 00000100:00000001:1.0:1788205084.509845:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321443348 00000100:00000001:1.0:1788205084.509847:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321444613 00000100:00000001:1.0:1788205084.509849:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205084.509851:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.509853:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205084.509856:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205084/real 1788205084] req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205084.509864:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205084.509865:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205084.509867:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205084.509869:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205084.509871:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205084.509872:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205084.509875:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205084.509883:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509889:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205084.509890:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509892:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.509893:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205084.509897:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509903:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205084.509907:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509913:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205084.509915:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509916:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.509917:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.509919:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205084.509921:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205084.509923:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205084.509926:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205084.509928:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205084.509932:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205084.509938:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509120:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205084.509942:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205084.509945:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509951:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205084.509952:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.509954:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205084.509956:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205084.509957:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205084.509960:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509248:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205084.509963:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205084.509965:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205084.509967:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.509968:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205084.509971:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205084.509973:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205084.509977:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205084.509979:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205084.509982:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205084.509984:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543509248, portal 4 00000100:00000001:1.0:1788205084.509986:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205084.509987:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788205084.509992:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.509999:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205084.510001:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205084.510003:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205084.510005:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543509248, offset 0 00000400:00000200:1.0:1788205084.510008:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205084.510012:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.510014:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205084.510017:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205084.510019:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205084.510021:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205084.510023:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205084.510025:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471327:471327:256:4294967295] 192.168.201.118@tcp LPNI seq info [471186:471186:8:4294967295] 00000400:00000200:1.0:1788205084.510028:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.510033:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205084.510037:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205084.510040:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205084.510043:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205084.510045:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205084.510048:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205084.510050:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205084.510054:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205084.510055:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205084.510058:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205084.510061:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205084.510064:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205084.510065:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205084.510067:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205084.510070:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510076:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.510079:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510085:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.510087:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205084.510112:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205084.510115:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205084.510118:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205084.510120:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321443358 00000100:00000001:1.0:1788205084.510123:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321444613 00000100:00000001:1.0:1788205084.510125:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321443358 00000100:00000001:1.0:1788205084.510126:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321444613 00000100:00000001:1.0:1788205084.510128:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205084.510130:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.510132:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205084.510135:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205084/real 1788205084] req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205084.510142:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205084.510143:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205084.510145:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205084.510147:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205084.510148:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205084.510150:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205084.510152:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205084.510159:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510165:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205084.510166:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510167:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.510168:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205084.510171:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510177:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205084.510180:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510186:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205084.510188:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510189:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.510190:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510192:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205084.510193:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205084.510195:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205084.510198:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205084.510200:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205084.510203:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205084.510210:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509248:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205084.510212:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205084.510215:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510220:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205084.510222:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.510223:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205084.510225:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205084.510226:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205084.510228:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509376:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205084.510231:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205084.510233:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205084.510234:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205084.510235:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205084.510238:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205084.510240:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205084.510243:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205084.510245:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205084.510247:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205084.510250:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543509376, portal 4 00000100:00000001:1.0:1788205084.510251:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205084.510252:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205084.510257:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510262:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205084.510264:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205084.510266:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205084.510268:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543509376, offset 0 00000400:00000200:1.0:1788205084.510271:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205084.510274:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.510277:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205084.510279:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205084.510281:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205084.510283:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205084.510285:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205084.510289:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471328:471328:256:4294967295] 192.168.201.118@tcp LPNI seq info [471187:471187:8:4294967295] 00000400:00000200:1.0:1788205084.510292:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205084.510298:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205084.510302:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205084.510304:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205084.510308:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205084.510310:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205084.510314:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205084.510316:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205084.510334:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205084.510335:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205084.510338:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205084.510341:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205084.510344:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205084.510345:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205084.510347:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205084.510350:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510357:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.510359:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205109 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510364:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.510366:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205084.510368:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205084.510371:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205084.510374:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205084.510376:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321443381 00000100:00000001:1.0:1788205084.510378:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321444613 00000100:00000001:1.0:1788205084.510380:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321443381 00000100:00000001:1.0:1788205084.510381:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321444613 00000100:00000001:1.0:1788205084.510383:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205084.510385:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.510386:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205084.510389:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205084/real 1788205084] req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205084.510395:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205084.510396:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205084.510398:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205084.510399:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205084.510401:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205084.510403:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205084.510406:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205084.510413:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510423:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205084.510424:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510426:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.510427:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205084.510430:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510437:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205084.510440:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510445:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205084.510447:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510448:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205084.510450:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510451:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205084.510453:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205084.510455:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205084.510458:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205084.510460:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205084.510463:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205084.510470:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509376:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205084.510473:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510475:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.510477:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543508992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510483:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205084.510484:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205084.510487:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205084.510489:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205084.510491:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205084.510492:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205084.510494:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205084.510496:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788205084.510499:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205084.510500:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205084.510501:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205084.510504:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205084.510505:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510506:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.510509:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543509120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510516:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205084.510517:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205084.510518:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205084.510520:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205084.510522:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205084.510523:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205084.510524:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205084.510525:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788205084.510527:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205084.510528:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205084.510529:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205084.510531:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205084.510532:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510534:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.510536:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543509248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510541:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205084.510542:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205084.510544:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205084.510545:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205084.510546:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205084.510547:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205084.510549:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205084.510550:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205084.510552:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205084.510553:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205084.510555:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205084.510556:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205084.510557:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510559:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205084.510562:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543509376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205109 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205084.510568:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205084.510569:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205084.510571:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205084.510573:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205084.510574:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205084.510575:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205084.510577:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205084.510578:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205084.510580:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205084.510581:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205084.510582:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205084.510584:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205084.510585:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510587:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205084.510592:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205084.510593:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205084.510595:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205084.510597:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205085.340177:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205085.340177:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205085.340181:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205085.340184:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205085.340184:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205085.340185:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205085.340187:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205085.340187:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205085.340191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205085.340193:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205085.340193:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205085.340194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205085.340200:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205085.340200:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205085.340201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205085.340204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205085.340205:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205085.340206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205085.340207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205085.340210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205085.340212:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205085.340212:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205085.340217:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205085.340217:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205085.340223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205085.340223:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205085.340262:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205085.340266:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205085.340270:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205085.852190:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321444613, at 4321444672 00000100:00000040:3.0:1788205085.852190:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321444613, at 4321444672 00000100:00000010:3.0:1788205085.852207:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00000100:00000010:2.0:1788205085.852209:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c00. 00000100:00000001:1.0:1788205086.364186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205086.364186:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205086.364188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205086.364193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205086.364193:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205086.364195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205086.364196:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205086.364199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205086.364199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205086.364201:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205086.364202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205086.364205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205086.364206:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205086.364207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205086.364209:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205086.364210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205086.364211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205086.364212:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205086.364215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205086.364216:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205086.364217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205086.364218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205086.364223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205086.364224:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205086.364228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205086.364232:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205086.364320:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205086.364324:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205086.364327:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205087.388171:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205087.388177:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205087.388178:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205087.388179:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.388181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205087.388182:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205087.388184:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205087.388186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205087.388189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205087.388189:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205087.388190:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205087.388193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205087.388193:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205087.388194:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205087.388200:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205087.388200:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205087.388201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205087.388205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205087.388205:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205087.388206:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205087.388208:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.388211:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205087.388212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205087.388214:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205087.388215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205087.388219:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205087.388267:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205087.388280:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205087.388284:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205088.412183:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205088.412183:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205088.412186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205088.412190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205088.412190:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205088.412191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205088.412193:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205088.412195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205088.412197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205088.412197:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205088.412200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205088.412201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205088.412202:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205088.412204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205088.412206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205088.412206:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205088.412207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205088.412210:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205088.412211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205088.412212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205088.412214:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205088.412215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205088.412217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205088.412220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205088.412223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205088.412223:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205088.412280:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205088.412283:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205088.412287:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205089.436169:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205089.436169:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205089.436177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205089.436177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205089.436177:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205089.436180:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.436181:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205089.436183:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205089.436187:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205089.436188:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205089.436191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205089.436193:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205089.436194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205089.436194:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205089.436198:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205089.436198:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205089.436200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205089.436204:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205089.436206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205089.436206:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205089.436207:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205089.436210:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.436213:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205089.436215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205089.436215:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205089.436219:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205089.436282:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205089.436285:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205089.436288:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205089.628229:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205089.628250:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205089.628258:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205089.628261:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205089.628265:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205089.628271:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26776 00000400:00000200:3.0:1788205089.628279:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205089.628287:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205089.628290:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205089.628294:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628298:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205089.628301:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628304:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205089.628308:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205089.628311:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205089.628314:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205089.628317:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205089.628320:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205089.628321:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205089.628324:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628325:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205089.628326:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628327:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205089.628329:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205089.628332:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628339:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205089.628341:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205089.628344:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205089.628346:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.628347:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205089.628350:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205089.628358:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788205089.628361:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205089.628364:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205089.628365:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.628368:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205089.628370:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205089.628373:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205089.628379:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788205089.628384:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205089.628391:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205089.628398:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205089.628401:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205089.628404:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205089.628425:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205089.628431:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205089.628434:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205089.628436:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205089.628437:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205089.628440:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205089.628443:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26776 00000400:00000200:3.0:1788205089.628447:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205089.628450:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205089.628452:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205089.628455:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628457:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205089.628459:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628461:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205089.628463:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788205089.628463:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205089.628465:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205089.628466:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205089.628467:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1788205089.628468:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:1.0:1788205089.628470:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788205089.628470:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205089.628472:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205089.628475:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205089.628475:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628477:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205089.628479:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628481:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205089.628484:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1788205089.628484:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:1.0:1788205089.628487:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205089.628487:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628489:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.628490:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205089.628490:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205089.628493:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205089.628495:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205089.628495:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205089.628497:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205089.628498:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543509504:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205089.628499:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205089.628502:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1788205089.628503:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205089.628505:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.628506:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205089.628507:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.628508:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.628510:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1788205089.628510:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205089.628512:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205089.628514:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000010:1.0:1788205089.628515:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:3.0:1788205089.628516:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.628517:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205089.628519:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205089.628522:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205089.628527:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205089.628527:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205089.628531:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 02000000:00000010:3.0:1788205089.628531:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788205089.628534:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205089.628538:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00100000:3.0:1788205089.628539:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205089.628545:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543509504, portal 25 00000100:00000001:3.0:1788205089.628545:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205089.628547:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205089.628547:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628549:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:3.0:1788205089.628551:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205089.628554:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205089.628560:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628562:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788205089.628563:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1788205089.628564:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205089.628566:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:1.0:1788205089.628567:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205089.628570:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543509504, offset 0 00000100:00000001:3.0:1788205089.628573:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205089.628574:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788205089.628576:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628577:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205089.628579:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26776 00000400:00000200:3.0:1788205089.628581:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.628584:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205089.628584:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205089.628586:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205089.628588:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205089.628589:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788205089.628590:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205089.628592:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205089.628594:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788205089.628594:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205089.628596:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205089.628597:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:1.0:1788205089.628598:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000020:00000001:3.0:1788205089.628599:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1788205089.628601:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1788205089.628601:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000200:1.0:1788205089.628603:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205089.628607:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471329:471329:256:4294967295] 192.168.201.118@tcp LPNI seq info [471188:471188:8:4294967295] 00000020:00000001:3.0:1788205089.628608:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205089.628610:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205089.628612:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205089.628613:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628614:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205089.628616:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628617:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205089.628620:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:1.0:1788205089.628621:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:3.0:1788205089.628622:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628625:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205089.628627:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205089.628627:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205089.628629:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1788205089.628631:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 02000000:00000001:3.0:1788205089.628631:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.628632:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205089.628634:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205089.628636:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205089.628637:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205089.628638:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205089.628641:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00000010:3.0:1788205089.628641:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788205089.628643:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205089.628645:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1788205089.628646:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:3.0:1788205089.628647:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.628648:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205089.628650:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1788205089.628650:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 02000000:00000001:3.0:1788205089.628651:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788205089.628655:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000010:3.0:1788205089.628655:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000800:00000010:1.0:1788205089.628657:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205089.628663:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1788205089.628663:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205089.628667:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00100000:3.0:1788205089.628667:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205089.628672:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205089.628674:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205089.628674:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205089.628676:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628678:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1788205089.628680:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205089.628687:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205089.628687:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205089.628690:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205089.628693:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205089.628695:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205089.628695:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205089.628696:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1788205089.628698:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205089.628699:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205089.628701:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26776 00000400:00000200:3.0:1788205089.628704:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205089.628705:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628707:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1788205089.628707:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205089.628709:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205089.628712:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205089.628713:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.628714:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1788205089.628716:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205089.628716:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628718:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205089.628720:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321449733 00000020:00000001:3.0:1788205089.628720:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205089.628721:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205089.628722:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000010:1.0:1788205089.628724:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16475a0. 00000020:00000040:3.0:1788205089.628724:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205089.628726:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205089.628728:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205089.628730:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000040:3.0:1788205089.628730:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628732:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205089.628734:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205089.628735:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321449733 00000100:00000001:3.0:1788205089.628736:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000010:1.0:1788205089.628738:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16478a0. 00000020:00000040:3.0:1788205089.628738:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205089.628740:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205089.628741:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205089.628743:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628744:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205089.628744:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205089.628746:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205089.628747:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205089.628748:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628749:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788205089.628750:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:1.0:1788205089.628752:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205089/real 1788205089] req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788205089.628752:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205089.628755:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.628757:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205089.628760:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000400:00000001:1.0:1788205089.628762:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:3.0:1788205089.628762:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000400:00000001:1.0:1788205089.628763:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788205089.628764:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1788205089.628765:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 02000000:00000001:3.0:1788205089.628765:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205089.628767:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000100:00000001:3.0:1788205089.628767:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205089.628768:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1788205089.628769:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 02000000:00000001:3.0:1788205089.628770:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628771:0:256132:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000010:3.0:1788205089.628773:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000200:1.0:1788205089.628775:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205089.628776:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205089.628780:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205089.628783:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.628790:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205089.628792:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205089.628793:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205089.628794:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205089.628795:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628796:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205089.628798:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205089.628800:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205089.628806:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628807:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205089.628808:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1788205089.628811:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205089.628811:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205089.628813:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205089.628814:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205089.628817:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205089.628818:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205089.628819:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26776 00000100:00000001:1.0:1788205089.628820:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1788205089.628822:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205089.628824:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205089.628826:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205089.628828:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1788205089.628829:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205089.628830:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205089.628831:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205089.628832:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205089.628833:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1788205089.628833:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628834:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00080000:1.0:1788205089.628836:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1788205089.628837:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205089.628838:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205089.628839:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00080000:1.0:1788205089.628840:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000020:00000040:3.0:1788205089.628841:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788205089.628842:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1788205089.628843:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205089.628845:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205089.628846:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205089.628848:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628850:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205089.628852:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205089.628854:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00100000:1.0:1788205089.628855:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543509504:192.168.201.118@tcp:250:kworker.0 00000020:00000040:3.0:1788205089.628860:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205089.628863:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205089.628864:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205089.628865:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628866:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788205089.628867:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205089.628869:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543509504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205089.628870:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205089.628872:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205089.628874:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205089.628875:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205089.628876:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000008:00000001:3.0:1788205089.628877:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.628878:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788205089.628879:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205089.628881:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205089.628883:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000010:3.0:1788205089.628883:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000001:1.0:1788205089.628885:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205089.628885:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000040:1.0:1788205089.628887:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205089.628887:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205089.628888:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205089.628889:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000100:00000001:3.0:1788205089.628890:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205089.628891:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:1.0:1788205089.628892:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205089.628893:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.628895:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:3.0:1788205089.628896:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205089.628897:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205089.628898:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:3.0:1788205089.628898:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628901:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00100000:3.0:1788205089.628902:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205089.628903:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.628904:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205089.628908:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205089.628910:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.628911:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205089.628913:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205089.628913:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.628915:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205089.628916:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205089.628918:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788205089.628920:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205089.628921:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205089.628924:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26786) 00000100:00000001:1.0:1788205089.628927:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205089.628929:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.628932:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205089.628933:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205089.628935:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205089.628938:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509632:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205089.628941:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205089.628943:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205089.628945:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.628946:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205089.628949:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205089.628951:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205089.628954:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205089.628957:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205089.628960:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205089.628963:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543509632, portal 10 00000100:00000001:1.0:1788205089.628964:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205089.628965:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205089.628974:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.628983:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205089.628984:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205089.628986:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205089.628987:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543509632, offset 0 00000400:00000200:1.0:1788205089.628989:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205089.628994:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.628997:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205089.628999:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205089.629001:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205089.629002:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205089.629004:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205089.629006:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471330:471330:256:4294967295] 192.168.201.118@tcp LPNI seq info [471189:471189:8:4294967295] 00000400:00000200:1.0:1788205089.629008:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.629012:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205089.629016:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205089.629018:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205089.629020:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205089.629022:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788205089.629024:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205089.629026:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1788205089.629028:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205089.629030:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205089.629032:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205089.629034:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205089.629036:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205089.629037:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205089.629038:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205089.629040:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629045:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205089.629047:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629050:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629051:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205089.629053:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205089.629055:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205089.629057:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205089.629059:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321448466 00000100:00000001:1.0:1788205089.629061:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321449733 00000100:00000001:1.0:1788205089.629062:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321448466 00000100:00000001:1.0:1788205089.629063:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321449733 00000100:00000001:1.0:1788205089.629064:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205089.629065:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629066:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205089.629068:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205089/real 1788205089] req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205089.629073:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205089.629074:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205089.629075:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205089.629077:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205089.629082:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205089.629083:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205089.629085:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205089.629109:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629113:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205089.629114:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629115:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205089.629116:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205089.629118:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629122:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205089.629124:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629128:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205089.629129:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629130:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205089.629131:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629131:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205089.629133:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205089.629134:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205089.629136:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205089.629137:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205089.629140:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205089.629144:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509632:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205089.629146:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205089.629148:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629151:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205089.629152:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.629153:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205089.629154:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205089.629156:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205089.629157:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509760:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205089.629159:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205089.629160:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205089.629161:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.629162:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205089.629164:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205089.629165:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205089.629167:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205089.629169:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205089.629171:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205089.629172:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543509760, portal 10 00000100:00000001:1.0:1788205089.629173:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205089.629174:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205089.629177:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629181:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205089.629183:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205089.629184:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205089.629185:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543509760, offset 0 00000400:00000200:1.0:1788205089.629188:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205089.629195:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.629198:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205089.629199:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205089.629201:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205089.629202:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205089.629204:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205089.629206:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471331:471331:256:4294967295] 192.168.201.118@tcp LPNI seq info [471190:471190:8:4294967295] 00000400:00000200:1.0:1788205089.629208:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.629212:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205089.629215:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205089.629217:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205089.629219:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205089.629220:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788205089.629223:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205089.629224:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1788205089.629227:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205089.629228:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205089.629244:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205089.629247:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205089.629249:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205089.629250:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205089.629252:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205089.629254:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629258:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205089.629260:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629264:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629265:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205089.629266:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205089.629268:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205089.629270:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205089.629272:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321448468 00000100:00000001:1.0:1788205089.629278:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321449733 00000100:00000001:1.0:1788205089.629281:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321448468 00000100:00000001:1.0:1788205089.629283:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321449733 00000100:00000001:1.0:1788205089.629288:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205089.629290:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629291:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205089.629295:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205089/real 1788205089] req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205089.629302:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205089.629303:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205089.629305:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205089.629307:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205089.629308:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205089.629310:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205089.629313:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205089.629320:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629327:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205089.629328:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629330:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205089.629331:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205089.629334:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629340:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205089.629343:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629350:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205089.629353:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629354:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205089.629355:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629357:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205089.629359:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205089.629362:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205089.629364:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205089.629365:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205089.629367:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205089.629372:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509760:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205089.629374:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205089.629376:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629379:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205089.629380:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.629382:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205089.629383:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205089.629384:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205089.629386:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509888:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205089.629388:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205089.629389:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205089.629390:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.629391:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205089.629394:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205089.629395:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205089.629397:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205089.629399:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205089.629401:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205089.629403:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543509888, portal 4 00000100:00000001:1.0:1788205089.629405:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205089.629406:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205089.629409:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629413:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205089.629414:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205089.629416:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205089.629417:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543509888, offset 0 00000400:00000200:1.0:1788205089.629419:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205089.629422:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.629423:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205089.629425:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205089.629427:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205089.629428:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205089.629429:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205089.629431:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471332:471332:256:4294967295] 192.168.201.118@tcp LPNI seq info [471191:471191:8:4294967295] 00000400:00000200:1.0:1788205089.629434:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.629437:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205089.629440:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205089.629442:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205089.629444:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205089.629445:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788205089.629448:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205089.629449:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1788205089.629452:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205089.629453:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205089.629454:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205089.629456:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205089.629458:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205089.629459:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205089.629461:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205089.629463:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629471:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205089.629473:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629476:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629477:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205089.629479:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205089.629480:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205089.629482:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205089.629483:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321448477 00000100:00000001:1.0:1788205089.629484:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321449733 00000100:00000001:1.0:1788205089.629486:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321448477 00000100:00000001:1.0:1788205089.629487:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321449733 00000100:00000001:1.0:1788205089.629488:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205089.629489:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629490:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205089.629492:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205089/real 1788205089] req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205089.629496:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205089.629497:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205089.629498:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205089.629499:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205089.629500:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205089.629501:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205089.629503:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205089.629507:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629511:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205089.629512:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629513:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205089.629514:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205089.629516:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629519:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205089.629521:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629525:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205089.629526:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629527:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205089.629527:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629528:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205089.629529:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205089.629531:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205089.629532:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205089.629533:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205089.629535:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205089.629539:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543509888:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205089.629541:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629542:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205089.629544:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543509632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629547:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205089.629548:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205089.629549:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205089.629551:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205089.629552:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205089.629553:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205089.629554:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205089.629555:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205089.629557:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205089.629557:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205089.629558:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205089.629560:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205089.629561:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629562:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205089.629564:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543509760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629568:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205089.629568:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205089.629569:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205089.629570:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205089.629571:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205089.629571:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205089.629572:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205089.629573:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205089.629574:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205089.629574:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205089.629575:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205089.629576:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205089.629577:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629578:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205089.629579:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543509888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629582:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205089.629583:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205089.629583:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205089.629584:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205089.629585:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205089.629586:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205089.629586:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205089.629587:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788205089.629588:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205089.629589:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205089.629590:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205089.629591:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205089.629592:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629593:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629595:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205089.629596:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629597:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205089.629598:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205089.629599:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205089.629601:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629605:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205089.629606:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.629607:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205089.629608:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205089.629609:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205089.629610:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510016:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205089.629612:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205089.629613:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205089.629614:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205089.629614:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205089.629616:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205089.629617:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205089.629618:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205089.629620:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205089.629621:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205089.629623:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543510016, portal 4 00000100:00000001:1.0:1788205089.629624:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205089.629625:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205089.629628:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629631:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205089.629632:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205089.629634:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205089.629635:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543510016, offset 0 00000400:00000200:1.0:1788205089.629637:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205089.629639:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.629641:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205089.629643:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205089.629644:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205089.629645:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205089.629647:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205089.629648:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471333:471333:256:4294967295] 192.168.201.118@tcp LPNI seq info [471192:471192:8:4294967295] 00000400:00000200:1.0:1788205089.629651:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205089.629654:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205089.629656:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205089.629658:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205089.629660:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205089.629662:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1788205089.629664:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205089.629665:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1788205089.629668:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205089.629669:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205089.629670:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205089.629672:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205089.629674:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205089.629675:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205089.629676:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205089.629678:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629682:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205089.629684:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205114 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629687:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629688:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205089.629690:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205089.629691:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205089.629693:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205089.629694:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321448500 00000100:00000001:1.0:1788205089.629696:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321449733 00000100:00000001:1.0:1788205089.629697:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321448500 00000100:00000001:1.0:1788205089.629698:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321449733 00000100:00000001:1.0:1788205089.629699:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205089.629700:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629701:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205089.629703:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205089/real 1788205089] req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205089.629707:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205089.629708:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205089.629709:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205089.629710:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205089.629711:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205089.629712:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205089.629714:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205089.629718:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629722:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205089.629723:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629724:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205089.629724:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205089.629726:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629730:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205089.629732:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629736:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205089.629737:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629738:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205089.629739:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629740:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205089.629741:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205089.629742:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205089.629744:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205089.629745:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205089.629747:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205089.629751:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510016:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205089.629752:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629753:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205089.629755:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543510016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205114 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205089.629758:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205089.629759:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205089.629760:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205089.629761:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205089.629762:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205089.629763:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205089.629764:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205089.629765:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205089.629766:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205089.629766:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205089.629767:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205089.629769:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205089.629769:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629771:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205089.629772:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205089.629773:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205089.629774:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205089.629776:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205090.460177:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205090.460180:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205090.460181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205090.460183:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205090.460185:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205090.460186:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205090.460186:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205090.460189:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205090.460192:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205090.460193:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205090.460196:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205090.460199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205090.460199:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205090.460200:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205090.460204:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205090.460205:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205090.460205:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205090.460209:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205090.460210:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205090.460210:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205090.460212:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205090.460215:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205090.460215:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205090.460227:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205090.460231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205090.460233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205090.460236:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205090.460237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205090.460240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205090.460241:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205090.460243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205090.460245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205090.460295:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205090.460300:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205090.460303:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205090.972192:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321449733, at 4321449792 00000100:00000040:3.0:1788205090.972192:0:417191:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321449733, at 4321449792 00000100:00000010:3.0:1788205090.972221:0:417191:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16478a0. 00000100:00000010:2.0:1788205090.972224:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00000100:00000001:2.0:1788205091.484163:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205091.484164:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205091.484165:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205091.484170:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205091.484172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205091.484173:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205091.484175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205091.484175:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205091.484178:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205091.484180:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205091.484182:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205091.484184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205091.484185:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205091.484187:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205091.484188:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205091.484189:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205091.484191:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205091.484194:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205091.484196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205091.484196:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205091.484197:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205091.484209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205091.484211:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205091.484215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205091.484217:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205091.484218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205091.484287:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205091.484295:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205091.484299:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205092.508200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205092.508200:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205092.508207:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205092.508209:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205092.508214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788205092.508214:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205092.508219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205092.508221:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788205092.508223:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205092.508224:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205092.508227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205092.508227:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205092.508228:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205092.508231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205092.508232:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205092.508237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205092.508238:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205092.508238:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205092.508242:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205092.508249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205092.508252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205092.508255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205092.508260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205092.508262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205092.508269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205092.508271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205092.508334:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205092.508338:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205092.508340:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205093.532232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205093.532236:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205093.532240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205093.532241:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205093.532246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205093.532246:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205093.532247:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205093.532254:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205093.532256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205093.532267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788205093.532268:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205093.532274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205093.532275:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205093.532275:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205093.532282:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205093.532283:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205093.532283:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205093.532288:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205093.532291:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205093.532292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205093.532295:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205093.532296:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205093.532299:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205093.532305:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532308:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205093.532313:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532317:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532321:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532325:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205093.532328:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205093.532329:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205093.532330:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532333:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205093.532335:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532336:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532338:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532346:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205093.532348:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205093.532350:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205093.532352:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532353:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205093.532354:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532356:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205093.532358:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205093.532433:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205093.532438:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205093.532441:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205094.556231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205094.556231:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205094.556239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205094.556242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205094.556242:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205094.556247:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.556247:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205094.556249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205094.556256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205094.556256:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205094.556263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205094.556264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205094.556264:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205094.556268:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.556269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205094.556270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.556273:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205094.556275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205094.556278:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205094.556279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205094.556279:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205094.556281:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205094.556283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.556284:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205094.556290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205094.556292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205094.556365:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205094.556369:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205094.556373:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205094.748261:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205094.748274:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205094.748277:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205094.748281:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205094.748287:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205094.748296:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26781 00000400:00000200:2.0:1788205094.748310:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205094.748319:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205094.748324:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205094.748327:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748331:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.748333:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748336:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205094.748341:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205094.748344:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205094.748348:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205094.748352:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205094.748354:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205094.748357:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205094.748361:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748363:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.748365:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748367:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205094.748371:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205094.748374:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748377:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205094.748380:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205094.748384:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205094.748388:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.748390:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205094.748394:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205094.748412:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788205094.748417:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205094.748421:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205094.748423:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.748427:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205094.748431:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205094.748437:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205094.748447:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788205094.748455:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205094.748466:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205094.748479:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205094.748485:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205094.748490:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205094.748523:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205094.748526:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205094.748530:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205094.748532:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205094.748533:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205094.748537:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205094.748540:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26781 00000400:00000200:2.0:1788205094.748544:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205094.748547:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205094.748550:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205094.748553:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748556:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.748558:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748560:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205094.748562:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205094.748564:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:1.0:1788205094.748572:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788205094.748573:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205094.748576:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788205094.748578:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788205094.748581:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205094.748585:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788205094.748586:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205094.748590:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205094.748591:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.748595:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748598:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788205094.748601:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205094.748602:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788205094.748609:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748614:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.748621:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788205094.748621:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205094.748625:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205094.748628:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205094.748631:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205094.748632:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.748637:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205094.748637:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205094.748642:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788205094.748643:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205094.748648:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543510144:192.168.201.118@tcp:250:kworker.0 00000008:00000001:2.0:1788205094.748649:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.748653:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.748657:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1788205094.748659:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788205094.748663:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205094.748665:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788205094.748665:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205094.748669:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.748670:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.748671:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205094.748674:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205094.748676:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788205094.748677:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205094.748683:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 02000000:00000010:1.0:1788205094.748686:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000001:2.0:1788205094.748687:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.748690:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205094.748695:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205094.748701:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205094.748709:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00000001:2.0:1788205094.748709:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1788205094.748720:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:2.0:1788205094.748725:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205094.748730:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543510144, portal 25 00000100:00000040:2.0:1788205094.748731:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.748736:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205094.748740:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000100:00000001:2.0:1788205094.748746:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205094.748751:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1788205094.748752:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205094.748759:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788205094.748763:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205094.748765:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205094.748770:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205094.748775:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205094.748781:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788205094.748781:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26781 00000400:00000010:1.0:1788205094.748787:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000400:00000200:2.0:1788205094.748789:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205094.748794:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000200:1.0:1788205094.748795:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543510144, offset 0 00000100:00080000:2.0:1788205094.748798:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205094.748803:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205094.748808:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748808:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.748813:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748816:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205094.748819:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205094.748821:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:1.0:1788205094.748824:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:2.0:1788205094.748825:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205094.748829:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788205094.748832:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205094.748836:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:2.0:1788205094.748836:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788205094.748843:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205094.748846:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1788205094.748847:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.748851:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205094.748854:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:2.0:1788205094.748855:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205094.748859:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:1.0:1788205094.748861:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1788205094.748864:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:1.0:1788205094.748868:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.748870:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205094.748873:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205094.748876:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205094.748877:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471334:471334:256:4294967295] 192.168.201.118@tcp LPNI seq info [471193:471193:8:4294967295] 00000400:00000200:1.0:1788205094.748885:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788205094.748888:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.748891:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205094.748893:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205094.748898:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.748901:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205094.748903:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000010:2.0:1788205094.748906:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788205094.748911:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000800:00000200:1.0:1788205094.748914:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:2.0:1788205094.748915:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205094.748918:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.748921:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205094.748922:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:1.0:1788205094.748923:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 02000000:00000001:2.0:1788205094.748925:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205094.748931:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000800:00000200:1.0:1788205094.748936:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788205094.748938:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205094.748943:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00100000:2.0:1788205094.748946:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205094.748954:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205094.748957:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788205094.748963:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788205094.748964:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1788205094.748973:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000100:00000001:2.0:1788205094.748975:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205094.748980:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788205094.748983:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788205094.748985:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000010:1.0:1788205094.748988:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00000001:2.0:1788205094.748988:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205094.748992:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205094.748995:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205094.748999:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1788205094.748999:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26781 00000400:00000200:2.0:1788205094.749007:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.749011:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1788205094.749014:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205094.749018:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1788205094.749021:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00000040:2.0:1788205094.749021:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000010:1.0:1788205094.749026:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:2.0:1788205094.749026:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.749030:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.749033:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205094.749035:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1788205094.749036:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205094.749039:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000100:00000200:1.0:1788205094.749042:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205094.749042:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205094.749045:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788205094.749049:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205094.749054:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788205094.749061:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205094.749062:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788205094.749064:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.749070:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205094.749071:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205094.749072:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205094.749079:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000001:1.0:1788205094.749083:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205094.749085:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205094.749086:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1788205094.749114:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205094.749117:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205094.749120:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205094.749123:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205094.749123:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.749125:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205094.749130:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00100000:2.0:1788205094.749130:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:1.0:1788205094.749135:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321454854 00000008:00000001:2.0:1788205094.749136:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205094.749140:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647ba0. 00000100:00000001:2.0:1788205094.749140:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749159:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000010:2.0:1788205094.749161:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788205094.749165:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788205094.749167:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321454854 02000000:00000001:2.0:1788205094.749168:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000010:1.0:1788205094.749172:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647e40. 02000000:00000001:2.0:1788205094.749172:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205094.749175:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205094.749176:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:2.0:1788205094.749179:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205094.749182:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205094.749187:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:1.0:1788205094.749189:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205094.749192:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749194:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749200:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788205094.749200:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205094.749209:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205094/real 1788205094] req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205094.749212:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205094.749217:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205094.749226:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205094.749229:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205094.749233:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205094.749237:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000100:00000001:2.0:1788205094.749239:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205094.749242:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000100:00080000:2.0:1788205094.749243:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1788205094.749246:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00080000:2.0:1788205094.749249:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205094.749252:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788205094.749252:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205094.749256:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1788205094.749259:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205094.749261:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205094.749266:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26781 00000400:00000200:2.0:1788205094.749274:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:1.0:1788205094.749278:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205094.749282:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205094.749288:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205094.749292:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1788205094.749294:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205094.749296:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205094.749299:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205094.749300:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.749303:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205094.749304:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.749308:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00100000:1.0:1788205094.749311:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205094.749312:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205094.749315:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788205094.749318:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205094.749321:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788205094.749324:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788205094.749326:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788205094.749332:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205094.749334:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205094.749335:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205094.749339:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205094.749342:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788205094.749343:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205094.749346:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788205094.749352:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205094.749356:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749360:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788205094.749360:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205094.749364:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205094.749366:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.749370:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788205094.749370:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749373:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205094.749375:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205094.749377:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00100000:2.0:1788205094.749378:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205094.749381:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1788205094.749382:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205094.749388:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788205094.749388:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205094.749396:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000010:2.0:1788205094.749397:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000100:00000001:1.0:1788205094.749400:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:2.0:1788205094.749401:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205094.749405:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205094.749409:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205094.749410:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205094.749413:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205094.749417:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205094.749420:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205094.749426:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788205094.749429:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205094.749436:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788205094.749443:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543510144:192.168.201.118@tcp:250:kworker.0 00000100:00000001:2.0:1788205094.749444:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205094.749449:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749453:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205094.749456:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749457:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.749463:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543510144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205094.749475:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749479:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205094.749483:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1788205094.749483:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26791) 02000000:00000010:1.0:1788205094.749487:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205094.749492:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205094.749494:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205094.749496:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205094.749498:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205094.749501:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:1.0:1788205094.749506:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205094.749509:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205094.749511:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:1.0:1788205094.749515:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205094.749516:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.749519:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.749528:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205094.749531:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749533:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205094.749534:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205094.749536:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205094.749539:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749546:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205094.749547:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.749549:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205094.749551:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205094.749553:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205094.749556:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510272:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205094.749560:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205094.749562:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205094.749563:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.749565:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205094.749568:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205094.749570:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205094.749574:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205094.749577:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205094.749581:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205094.749584:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543510272, portal 10 00000100:00000001:1.0:1788205094.749586:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205094.749587:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000040:1.0:1788205094.749595:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749601:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205094.749604:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205094.749606:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205094.749609:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543510272, offset 0 00000400:00000200:1.0:1788205094.749613:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205094.749617:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.749621:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205094.749623:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205094.749626:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205094.749628:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205094.749631:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205094.749634:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471335:471335:256:4294967295] 192.168.201.118@tcp LPNI seq info [471194:471194:8:4294967295] 00000400:00000200:1.0:1788205094.749638:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.749644:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205094.749649:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205094.749652:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205094.749656:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205094.749659:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205094.749663:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205094.749666:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205094.749670:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205094.749671:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205094.749674:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205094.749678:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205094.749681:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205094.749683:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205094.749685:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205094.749688:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749696:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.749699:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749706:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749707:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205094.749711:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205094.749715:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205094.749718:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205094.749720:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321453587 00000100:00000001:1.0:1788205094.749723:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321454854 00000100:00000001:1.0:1788205094.749725:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321453587 00000100:00000001:1.0:1788205094.749727:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321454854 00000100:00000001:1.0:1788205094.749729:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205094.749731:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749733:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205094.749736:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205094/real 1788205094] req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205094.749744:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205094.749745:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205094.749747:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205094.749749:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205094.749751:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205094.749753:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205094.749756:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205094.749765:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749771:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205094.749772:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.749774:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.749775:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205094.749778:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749785:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205094.749788:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749795:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205094.749798:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.749799:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.749800:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.749802:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205094.749804:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205094.749806:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205094.749808:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205094.749810:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205094.749813:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205094.749821:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510272:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205094.749825:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205094.749828:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749835:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205094.749836:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.749838:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205094.749840:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205094.749842:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205094.749845:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510400:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205094.749849:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205094.749851:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205094.749852:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.749854:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205094.749857:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205094.749859:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205094.749863:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205094.749865:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205094.749868:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205094.749871:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543510400, portal 10 00000100:00000001:1.0:1788205094.749873:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205094.749874:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 00000100:00000040:1.0:1788205094.749879:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749886:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205094.749890:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205094.749892:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205094.749894:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543510400, offset 0 00000400:00000200:1.0:1788205094.749900:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205094.749904:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.749908:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205094.749912:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205094.749914:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205094.749917:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205094.749920:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205094.749923:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471336:471336:256:4294967295] 192.168.201.118@tcp LPNI seq info [471195:471195:8:4294967295] 00000400:00000200:1.0:1788205094.749927:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.749933:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205094.749938:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205094.749941:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205094.749944:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205094.749947:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205094.749951:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205094.749954:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205094.749957:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205094.749959:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205094.749962:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205094.749965:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205094.749968:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205094.749971:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205094.749973:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205094.749976:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749985:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.749988:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.749996:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.749998:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205094.750001:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205094.750003:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205094.750007:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205094.750009:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321453588 00000100:00000001:1.0:1788205094.750012:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321454854 00000100:00000001:1.0:1788205094.750014:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321453588 00000100:00000001:1.0:1788205094.750016:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321454854 00000100:00000001:1.0:1788205094.750018:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205094.750020:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.750021:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205094.750024:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205094/real 1788205094] req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205094.750031:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205094.750032:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205094.750034:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205094.750035:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205094.750037:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205094.750039:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205094.750041:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205094.750050:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750057:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205094.750058:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750060:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.750061:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205094.750065:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750071:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205094.750074:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750080:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205094.750082:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750084:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.750085:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750086:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205094.750088:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205094.750106:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205094.750109:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205094.750111:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205094.750114:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205094.750122:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510400:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205094.750125:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205094.750129:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750135:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205094.750136:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.750138:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205094.750139:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205094.750141:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205094.750156:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510528:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205094.750158:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205094.750160:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205094.750161:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.750162:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205094.750164:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205094.750166:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205094.750169:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205094.750170:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205094.750173:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205094.750175:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543510528, portal 4 00000100:00000001:1.0:1788205094.750176:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205094.750177:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331919360 : -109424377632256 : ffff9c7aa6d76a00) 00000100:00000040:1.0:1788205094.750181:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750188:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205094.750190:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205094.750192:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205094.750194:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543510528, offset 0 00000400:00000200:1.0:1788205094.750198:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205094.750201:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.750205:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205094.750207:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205094.750210:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205094.750212:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205094.750214:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205094.750217:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471337:471337:256:4294967295] 192.168.201.118@tcp LPNI seq info [471196:471196:8:4294967295] 00000400:00000200:1.0:1788205094.750221:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.750227:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205094.750232:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205094.750235:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205094.750238:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205094.750240:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205094.750245:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205094.750247:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205094.750251:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205094.750253:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205094.750256:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205094.750259:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205094.750262:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205094.750263:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205094.750266:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205094.750270:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750277:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.750280:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750286:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.750288:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205094.750291:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205094.750294:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205094.750297:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205094.750299:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321453598 00000100:00000001:1.0:1788205094.750302:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321454854 00000100:00000001:1.0:1788205094.750304:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321453598 00000100:00000001:1.0:1788205094.750306:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321454854 00000100:00000001:1.0:1788205094.750308:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205094.750310:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.750312:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205094.750315:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205094/real 1788205094] req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205094.750323:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205094.750325:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205094.750327:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205094.750329:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205094.750331:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205094.750333:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205094.750336:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205094.750344:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750351:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205094.750352:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750354:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.750355:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205094.750359:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750366:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205094.750369:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750377:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205094.750379:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750381:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.750382:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750385:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205094.750386:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205094.750389:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205094.750392:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205094.750394:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205094.750399:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205094.750406:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510528:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205094.750410:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205094.750413:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750419:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205094.750421:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.750422:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205094.750424:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205094.750425:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205094.750426:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510656:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205094.750429:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205094.750430:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205094.750431:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205094.750433:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205094.750435:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205094.750437:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205094.750438:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205094.750440:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205094.750443:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205094.750445:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543510656, portal 4 00000100:00000001:1.0:1788205094.750446:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205094.750447:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331914880 : -109424377636736 : ffff9c7aa6d75880) 00000100:00000040:1.0:1788205094.750451:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750456:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205094.750457:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205094.750458:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205094.750460:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543510656, offset 0 00000400:00000200:1.0:1788205094.750463:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205094.750465:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.750467:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205094.750469:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205094.750471:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205094.750472:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205094.750474:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205094.750476:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471338:471338:256:4294967295] 192.168.201.118@tcp LPNI seq info [471197:471197:8:4294967295] 00000400:00000200:1.0:1788205094.750478:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205094.750482:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205094.750485:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205094.750486:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205094.750489:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205094.750490:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205094.750493:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205094.750495:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205094.750497:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205094.750498:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205094.750500:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205094.750502:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205094.750504:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205094.750505:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205094.750507:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205094.750509:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750514:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.750515:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205119 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750519:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.750520:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205094.750522:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205094.750524:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205094.750526:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205094.750527:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321453621 00000100:00000001:1.0:1788205094.750529:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321454854 00000100:00000001:1.0:1788205094.750531:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321453621 00000100:00000001:1.0:1788205094.750532:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321454854 00000100:00000001:1.0:1788205094.750533:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205094.750534:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.750535:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205094.750537:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205094/real 1788205094] req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205094.750541:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205094.750542:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205094.750543:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205094.750544:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205094.750545:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205094.750547:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205094.750548:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205094.750553:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750557:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205094.750557:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750558:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.750559:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205094.750561:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750564:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205094.750566:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750569:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205094.750571:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750572:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205094.750573:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750574:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205094.750575:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205094.750577:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205094.750579:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205094.750581:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205094.750584:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205094.750590:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510656:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205094.750594:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750596:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.750598:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543510272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750605:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205094.750607:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205094.750608:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205094.750611:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205094.750612:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205094.750613:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205094.750615:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205094.750618:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:1.0:1788205094.750621:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205094.750622:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205094.750624:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:1.0:1788205094.750627:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205094.750628:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750630:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.750633:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543510400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750644:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205094.750645:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205094.750646:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205094.750647:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205094.750648:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205094.750648:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205094.750650:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205094.750651:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:1.0:1788205094.750653:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205094.750653:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205094.750654:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:1.0:1788205094.750656:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205094.750656:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750657:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.750659:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76a00 x1875055543510528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750662:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205094.750663:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205094.750664:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205094.750666:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205094.750666:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205094.750667:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205094.750668:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205094.750669:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:1.0:1788205094.750670:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205094.750671:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205094.750671:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76a00. 00000100:00000001:1.0:1788205094.750673:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205094.750673:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750674:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205094.750675:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75880 x1875055543510656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205119 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205094.750679:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205094.750679:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205094.750681:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205094.750682:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205094.750683:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205094.750684:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205094.750686:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205094.750687:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d800. 02000000:00000001:1.0:1788205094.750689:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205094.750690:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205094.750691:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75880. 00000100:00000001:1.0:1788205094.750694:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205094.750695:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750697:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205094.750701:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205094.750702:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205094.750704:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205094.750705:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205095.580240:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205095.580240:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205095.580252:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205095.580252:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205095.580255:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205095.580255:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205095.580263:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205095.580264:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205095.580270:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205095.580271:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205095.580277:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205095.580277:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205095.580284:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205095.580284:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205095.580289:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205095.580290:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205095.580297:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205095.580297:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205095.580417:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205095.580421:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205095.580427:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205095.580428:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205095.580431:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205095.580433:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205095.580437:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205095.580438:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205095.580512:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205095.580516:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205095.580520:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205096.092271:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321454854, at 4321454912 00000100:00000040:3.0:1788205096.092271:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321454854, at 4321454912 00000100:00000010:3.0:1788205096.092294:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00000100:00000010:2.0:1788205096.092301:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647ba0. 00000100:00000001:1.0:1788205096.604200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205096.604200:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205096.604207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205096.604209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205096.604209:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205096.604213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205096.604213:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205096.604214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205096.604221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205096.604222:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205096.604223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205096.604227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205096.604228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205096.604228:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205096.604231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205096.604232:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205096.604233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205096.604233:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205096.604237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205096.604239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205096.604240:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205096.604241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205096.604242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205096.604246:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205096.604246:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205096.604252:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205096.604305:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205096.604310:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205096.604313:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205097.628220:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205097.628220:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205097.628228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205097.628231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205097.628231:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205097.628233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205097.628235:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205097.628236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205097.628244:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205097.628245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205097.628245:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205097.628248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205097.628254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205097.628254:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205097.628254:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205097.628258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205097.628261:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205097.628261:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205097.628263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205097.628265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205097.628269:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205097.628270:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205097.628275:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205097.628276:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205097.628282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205097.628282:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205097.628332:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205097.628336:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205097.628339:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205098.652173:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205098.652180:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205098.652183:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205098.652184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205098.652189:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205098.652190:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205098.652195:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205098.652198:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205098.652199:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205098.652201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205098.652204:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205098.652206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205098.652208:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205098.652209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205098.652213:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205098.652215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205098.652217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205098.652318:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205098.652321:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205098.652324:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205098.652476:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205098.652479:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205098.652481:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205098.652483:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205098.652484:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205098.652486:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205098.652487:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205098.652489:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205098.652490:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.676199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205099.676201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205099.676208:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205099.676208:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205099.676210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205099.676213:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205099.676217:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205099.676220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205099.676222:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205099.676226:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.676226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205099.676229:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205099.676230:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.676232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205099.676235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205099.676236:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.676237:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205099.676244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205099.676247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205099.676250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205099.676251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205099.676255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205099.676256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205099.676337:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205099.676340:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205099.676341:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205099.676344:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205099.676346:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205099.676349:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788205099.676351:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000100:00000001:2.0:1788205099.676352:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205099.676356:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788205099.676357:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205099.676360:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205099.676366:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205099.868221:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205099.868232:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205099.868235:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205099.868238:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205099.868243:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205099.868250:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26786 00000400:00000200:1.0:1788205099.868260:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205099.868266:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205099.868268:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205099.868271:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868273:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868275:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868277:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205099.868279:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205099.868281:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205099.868283:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205099.868286:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205099.868288:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205099.868290:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205099.868292:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868294:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868296:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868298:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205099.868301:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205099.868305:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868307:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205099.868310:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205099.868314:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205099.868316:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868319:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205099.868322:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205099.868332:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000020:00000040:1.0:1788205099.868334:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205099.868337:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205099.868338:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868341:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205099.868343:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205099.868346:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205099.868354:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb400. 00000100:00000001:1.0:1788205099.868360:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205099.868369:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205099.868380:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205099.868383:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205099.868389:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205099.868417:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205099.868420:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205099.868424:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205099.868425:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205099.868427:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205099.868429:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205099.868432:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26786 00000400:00000200:1.0:1788205099.868435:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205099.868437:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205099.868438:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205099.868440:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868442:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868443:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868444:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205099.868445:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205099.868446:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205099.868447:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205099.868449:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205099.868450:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205099.868451:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205099.868453:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868454:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868455:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868456:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205099.868457:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205099.868459:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868460:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205099.868461:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205099.868467:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205099.868468:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868469:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205099.868472:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205099.868474:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868476:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205099.868479:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205099.868482:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205099.868484:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205099.868485:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868487:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205099.868488:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205099.868489:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205099.868493:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba800. 00000100:00000001:1.0:1788205099.868496:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205099.868501:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205099.868509:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205099.868511:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205099.868515:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205099.868522:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205099.868524:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205099.868527:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205099.868528:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205099.868529:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205099.868531:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205099.868532:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26786 00000400:00000200:1.0:1788205099.868535:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205099.868536:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205099.868538:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205099.868539:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868541:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868542:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868543:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205099.868545:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205099.868545:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205099.868546:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205099.868547:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205099.868548:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205099.868549:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205099.868551:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868552:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868553:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868554:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205099.868556:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205099.868557:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868559:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205099.868560:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205099.868561:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205099.868562:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868563:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205099.868565:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205099.868566:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868567:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205099.868569:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000020:00000040:1.0:1788205099.868570:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205099.868572:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205099.868573:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868574:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205099.868575:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205099.868576:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205099.868578:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000001:1.0:1788205099.868580:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205099.868583:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205099.868587:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205099.868588:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205099.868590:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205099.868594:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205099.868596:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205099.868597:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205099.868598:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205099.868599:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205099.868600:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205099.868602:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26786 00000400:00000200:1.0:1788205099.868604:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205099.868605:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205099.868606:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205099.868608:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868609:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868610:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868611:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205099.868613:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205099.868614:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205099.868615:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205099.868617:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205099.868618:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205099.868620:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205099.868623:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868624:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868626:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868628:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205099.868630:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205099.868633:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868635:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205099.868637:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205099.868638:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205099.868640:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868642:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205099.868644:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205099.868646:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868648:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205099.868651:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205099.868653:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205099.868655:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205099.868656:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868658:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205099.868659:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205099.868661:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205099.868663:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000001:1.0:1788205099.868665:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205099.868668:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205099.868672:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205099.868673:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205099.868675:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205099.868680:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205099.868683:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205099.868685:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205099.868687:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205099.868689:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205099.868691:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205099.868693:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26786 00000400:00000200:1.0:1788205099.868696:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205099.868699:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205099.868701:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205099.868704:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868705:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868707:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868709:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205099.868711:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205099.868712:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205099.868714:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205099.868716:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205099.868718:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205099.868720:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205099.868723:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868724:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205099.868726:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868727:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205099.868728:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205099.868730:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205099.868731:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205099.868732:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205099.868734:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205099.868735:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868736:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205099.868738:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205099.868739:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868740:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205099.868744:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205099.868745:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205099.868747:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205099.868747:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205099.868748:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205099.868750:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205099.868751:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205099.868753:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8800. 00000100:00000001:1.0:1788205099.868755:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205099.868758:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205099.868762:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205099.868763:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205099.868765:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205099.868769:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205099.868772:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26796) 00000100:00000001:3.0:1788205099.868839:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205099.868843:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205099.868846:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205099.868851:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.868865:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205099.868866:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.868870:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205099.868872:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205099.868874:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205099.868878:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543510784:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205099.868881:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205099.868886:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205099.868900:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.868902:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205099.868907:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205099.868909:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205099.868914:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205099.868917:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205099.868925:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205099.868931:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543510784, portal 25 00000100:00000001:3.0:1788205099.868933:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205099.868935:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788205099.868940:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.868946:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205099.868949:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205099.868951:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205099.868954:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543510784, offset 0 00000400:00000200:3.0:1788205099.868960:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205099.868968:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.868973:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205099.868978:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205099.868982:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205099.868986:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205099.868988:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205099.869004:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471339:471339:256:4294967295] 192.168.201.118@tcp LPNI seq info [471198:471198:8:4294967295] 00000400:00000200:3.0:1788205099.869009:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.869018:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205099.869023:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.869036:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205099.869043:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205099.869047:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205099.869052:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205099.869086:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205099.869118:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.869120:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205099.869126:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205099.869130:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205099.869136:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205099.869138:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205099.869142:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205099.869147:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869157:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.869161:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869167:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.869169:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205099.869175:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205099.869178:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205099.869181:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321459974 00000100:00000010:3.0:1788205099.869185:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc7e0. 00000100:00000001:3.0:1788205099.869192:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205099.869197:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321459974 00000100:00000010:3.0:1788205099.869200:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc4e0. 00000100:00000001:3.0:1788205099.869204:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205099.869207:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205099.869210:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.869212:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205099.869216:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205099/real 1788205099] req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205099.869225:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205099.869227:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205099.869228:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205099.869231:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205099.869233:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205099.869235:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205099.869238:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205099.869247:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869253:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205099.869254:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.869257:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.869258:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205099.869262:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869269:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205099.869273:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869280:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205099.869283:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.869285:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.869286:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.869288:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205099.869290:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205099.869293:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205099.869296:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205099.869299:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205099.869303:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205099.869311:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543510784:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205099.869315:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205099.869318:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869324:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205099.869326:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.869328:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205099.869330:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205099.869331:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205099.869335:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510912:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205099.869338:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205099.869339:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205099.869341:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.869342:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205099.869346:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205099.869348:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205099.869351:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205099.869353:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205099.869357:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205099.869360:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543510912, portal 10 00000100:00000001:3.0:1788205099.869361:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205099.869363:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205099.869370:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869376:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205099.869378:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205099.869380:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205099.869383:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543510912, offset 0 00000400:00000200:3.0:1788205099.869386:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205099.869390:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.869394:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205099.869397:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205099.869400:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205099.869402:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205099.869404:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205099.869407:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471340:471340:256:4294967295] 192.168.201.118@tcp LPNI seq info [471199:471199:8:4294967295] 00000400:00000200:3.0:1788205099.869411:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.869417:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205099.869421:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.869425:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205099.869428:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205099.869431:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205099.869435:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205099.869437:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205099.869441:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.869443:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205099.869445:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205099.869449:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205099.869452:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205099.869454:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205099.869456:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205099.869460:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869467:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.869470:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869476:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.869477:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205099.869481:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205099.869484:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205099.869488:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205099.869490:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321458707 00000100:00000001:3.0:1788205099.869492:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321459974 00000100:00000001:3.0:1788205099.869495:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321458707 00000100:00000001:3.0:1788205099.869497:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321459974 00000100:00000001:3.0:1788205099.869499:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205099.869501:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.869503:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205099.869506:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205099/real 1788205099] req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205099.869513:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205099.869515:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205099.869516:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205099.869519:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205099.869520:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205099.869522:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205099.869525:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205099.869533:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869540:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205099.869541:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.869543:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.869544:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205099.869548:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869554:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205099.869558:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869565:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205099.869567:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.869569:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.869570:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.869572:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205099.869574:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205099.869576:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205099.869579:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205099.869581:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205099.869585:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205099.869592:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543510912:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205099.869596:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205099.869599:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869605:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205099.869607:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.869609:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205099.869610:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205099.869612:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205099.869615:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511040:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205099.869618:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205099.869620:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205099.869621:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.869622:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205099.869625:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205099.869627:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205099.869631:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205099.869633:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205099.869636:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205099.869639:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543511040, portal 10 00000100:00000001:3.0:1788205099.869641:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205099.869642:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205099.869647:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.869653:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205099.869655:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205099.869658:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205099.869660:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543511040, offset 0 00000400:00000200:3.0:1788205099.869664:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205099.869668:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.869671:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205099.869674:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205099.869676:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205099.869678:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205099.869680:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205099.869683:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471341:471341:256:4294967295] 192.168.201.118@tcp LPNI seq info [471200:471200:8:4294967295] 00000400:00000200:3.0:1788205099.869687:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.869693:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205099.869697:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.869700:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205099.869705:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205099.869708:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205099.869711:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205099.869714:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205099.869717:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.869719:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205099.869722:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205099.869725:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205099.869728:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205099.871004:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205099.871016:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205099.871036:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871050:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.871064:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871073:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.871074:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205099.871086:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205099.871109:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205099.871114:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205099.871118:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321458710 00000100:00000001:3.0:1788205099.871126:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321459974 00000100:00000001:3.0:1788205099.871129:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321458710 00000100:00000001:3.0:1788205099.871131:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321459974 00000100:00000001:3.0:1788205099.871133:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205099.871136:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.871139:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205099.871144:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205099/real 1788205099] req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205099.871155:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205099.871157:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205099.871168:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205099.871172:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205099.871174:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205099.871176:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205099.871179:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205099.871187:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871195:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205099.871197:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871199:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.871201:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205099.871206:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871213:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205099.871218:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871225:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205099.871228:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871230:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.871237:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871240:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205099.871243:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205099.871247:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205099.871251:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205099.871254:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205099.871258:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205099.871266:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511040:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205099.871271:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205099.871275:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871282:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205099.871284:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.871288:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205099.871291:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205099.871294:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205099.871297:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511168:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205099.871300:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205099.871305:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205099.871307:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.871310:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205099.871317:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205099.871320:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205099.871325:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205099.871330:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205099.871335:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205099.871339:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543511168, portal 4 00000100:00000001:3.0:1788205099.871342:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205099.871343:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205099.871350:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871362:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205099.871364:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205099.871366:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205099.871369:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543511168, offset 0 00000400:00000200:3.0:1788205099.871374:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205099.871381:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.871387:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205099.871392:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205099.871396:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205099.871400:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205099.871402:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205099.871406:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471342:471342:256:4294967295] 192.168.201.118@tcp LPNI seq info [471201:471201:8:4294967295] 00000400:00000200:3.0:1788205099.871411:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.871419:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205099.871426:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.871431:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205099.871438:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205099.871442:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205099.871448:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205099.871452:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205099.871456:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.871459:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205099.871463:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205099.871468:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205099.871472:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205099.871474:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205099.871476:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205099.871480:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871488:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.871491:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871497:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.871499:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205099.871502:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205099.871505:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205099.871509:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205099.871511:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321458719 00000100:00000001:3.0:1788205099.871513:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321459974 00000100:00000001:3.0:1788205099.871516:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321458719 00000100:00000001:3.0:1788205099.871517:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321459974 00000100:00000001:3.0:1788205099.871520:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205099.871522:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.871524:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205099.871527:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205099/real 1788205099] req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205099.871536:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205099.871537:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205099.871539:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205099.871541:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205099.871543:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205099.871545:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205099.871548:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205099.871556:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871562:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205099.871563:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871565:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.871566:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205099.871569:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871576:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205099.871580:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871586:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205099.871588:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871590:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.871591:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871593:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205099.871595:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205099.871598:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205099.871601:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205099.871603:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205099.871608:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205099.871616:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511168:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205099.871620:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205099.871624:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871631:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205099.871633:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.871635:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205099.871637:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205099.871639:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205099.871642:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511296:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205099.871645:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205099.871647:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205099.871649:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205099.871650:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205099.871654:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205099.871656:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205099.871659:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205099.871662:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205099.871665:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205099.871668:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543511296, portal 4 00000100:00000001:3.0:1788205099.871670:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205099.871671:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205099.871676:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871682:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205099.871684:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205099.871687:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000200:3.0:1788205099.871689:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543511296, offset 0 00000400:00000200:3.0:1788205099.871694:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205099.871698:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.871702:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205099.871705:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205099.871707:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205099.871710:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205099.871712:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205099.871715:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471343:471343:256:4294967295] 192.168.201.118@tcp LPNI seq info [471202:471202:8:4294967295] 00000400:00000200:3.0:1788205099.871719:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205099.871725:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205099.871730:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.871733:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205099.871736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205099.871739:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205099.871749:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205099.871752:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205099.871755:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205099.871757:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:3.0:1788205099.871760:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205099.871763:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205099.871766:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:3.0:1788205099.871768:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:3.0:1788205099.871770:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205099.871773:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871781:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.871784:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205124 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871790:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.871791:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205099.871794:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205099.871797:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205099.871800:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205099.871802:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321458742 00000100:00000001:3.0:1788205099.871805:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321459974 00000100:00000001:3.0:1788205099.871807:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321458742 00000100:00000001:3.0:1788205099.871809:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321459974 00000100:00000001:3.0:1788205099.871811:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205099.871820:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.871822:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205099.871826:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205099/real 1788205099] req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205099.871833:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205099.871834:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205099.871836:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205099.871838:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205099.871839:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205099.871841:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205099.871844:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205099.871851:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871857:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205099.871858:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871859:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.871860:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205099.871863:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871869:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205099.871873:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871878:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205099.871881:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871883:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205099.871884:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871886:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205099.871887:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205099.871890:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205099.871893:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205099.871895:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205099.871898:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205099.871905:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511296:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205099.871909:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871911:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.871914:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543510784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871921:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205099.871923:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205099.871926:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205099.871929:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205099.871931:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205099.871934:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205099.871936:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205099.871938:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:3.0:1788205099.871942:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205099.871944:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205099.871946:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205099.871954:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205099.871955:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871957:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.871961:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543510912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871973:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205099.871974:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205099.871975:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205099.871978:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205099.871979:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205099.871980:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205099.871982:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205099.871983:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:3.0:1788205099.871985:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205099.871986:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205099.871987:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205099.871989:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205099.871990:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.871991:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.871993:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543511040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.871998:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205099.871999:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205099.872000:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205099.872002:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205099.872003:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205099.872005:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205099.872007:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205099.872008:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:3.0:1788205099.872010:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205099.872011:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205099.872012:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205099.872018:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205099.872019:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.872020:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.872023:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543511168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.872028:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205099.872029:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205099.872031:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205099.872033:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205099.872034:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205099.872035:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205099.872037:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205099.872038:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:3.0:1788205099.872040:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205099.872040:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205099.872041:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205099.872043:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205099.872044:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.872045:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205099.872048:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543511296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205124 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205099.872180:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205099.872181:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205099.872183:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205099.872185:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205099.872186:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205099.872188:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205099.872189:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205099.872191:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:3.0:1788205099.872194:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205099.872195:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205099.872196:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205099.872201:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205099.872203:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.872205:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205099.872216:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205099.872218:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205099.872220:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205099.872223:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205100.700244:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205100.700254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205100.700256:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205100.700260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205100.700261:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205100.700267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205100.700268:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205100.700270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205100.700276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205100.700276:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205100.700278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205100.700282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205100.700282:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205100.700284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205100.700289:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205100.700293:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205100.700298:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205100.700376:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205100.700379:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205100.700388:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205100.700752:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205100.700757:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205100.700758:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205100.700761:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205100.700764:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205100.700766:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205100.700768:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205100.700771:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205100.700773:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205101.212262:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321459974, at 4321460032 00000100:00000010:3.0:1788205101.212282:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc4e0. 00000100:00000040:2.0:1788205101.212428:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321459974, at 4321460032 00000100:00000010:2.0:1788205101.212435:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc7e0. 00000100:00000001:1.0:1788205101.724203:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205101.724212:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205101.724214:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205101.724220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205101.724224:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205101.724228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205101.724230:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205101.724233:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205101.724236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205101.724246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205101.724251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205101.724265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205101.724267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205101.724271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205101.724272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205101.724277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205101.724278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205101.724340:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205101.724348:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205101.724351:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205101.724358:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205101.724362:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205101.724371:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205101.724374:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205101.724378:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788205101.724378:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000100:00000001:2.0:1788205101.724383:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205101.724383:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205101.724391:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205102.748210:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205102.748218:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205102.748220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205102.748225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205102.748228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205102.748232:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205102.748234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205102.748236:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205102.748239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205102.748414:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205102.748421:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205102.748423:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205102.748428:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205102.748431:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205102.748435:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205102.748438:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205102.748441:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205102.748444:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205102.749623:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205102.749627:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205102.749632:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205102.749633:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205102.749636:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205102.749638:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205102.749641:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205102.749642:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205102.749792:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205102.749797:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205102.749800:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205103.772218:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205103.772222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205103.772228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205103.772230:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205103.772232:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:1.0:1788205103.772236:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205103.772238:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205103.772241:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205103.772241:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205103.772244:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205103.772245:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205103.772247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205103.772248:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205103.772250:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205103.772251:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205103.772252:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205103.772254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772255:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205103.772260:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205103.772261:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205103.772262:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772264:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205103.772265:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772267:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772269:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205103.772275:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205103.772276:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205103.772277:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772279:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205103.772280:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772281:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205103.772283:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205103.772384:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205103.772388:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205103.772391:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205103.773048:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205103.773052:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205103.773053:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205103.773055:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205103.773057:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205103.773059:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205103.773061:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205103.773063:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205103.773064:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205104.796667:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205104.796687:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205104.796695:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205104.796696:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205104.796700:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205104.796701:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205104.796705:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205104.796706:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205104.796886:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205104.796891:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205104.796896:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205104.797208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205104.797216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205104.797218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205104.797223:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205104.797228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205104.797231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205104.797234:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205104.797237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205104.797241:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205104.797850:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205104.797855:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205104.797857:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205104.797861:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205104.797863:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205104.797865:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205104.797868:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205104.797871:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205104.797873:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205104.989267:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.989281:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205104.989350:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205104.989356:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205104.989361:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205104.989370:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26791 00000400:00000200:0.0:1788205104.989383:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205104.989393:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205104.989397:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205104.989400:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989403:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205104.989406:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989408:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205104.989414:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205104.989416:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205104.989419:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205104.989423:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205104.989427:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205104.989430:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205104.989432:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989434:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205104.989436:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989438:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205104.989441:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205104.989444:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989447:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205104.989450:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205104.989454:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205104.989457:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.989459:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205104.989463:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205104.989477:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788205104.989481:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205104.989485:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205104.989487:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.989491:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205104.989495:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205104.989500:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205104.989509:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788205104.989517:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205104.989528:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205104.989540:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205104.989546:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205104.989551:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205104.989582:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205104.989586:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.989590:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205104.989592:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205104.989593:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205104.989597:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205104.989600:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26791 00000400:00000200:0.0:1788205104.989604:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205104.989607:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205104.989609:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205104.989613:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989615:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205104.989617:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989619:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205104.989621:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205104.989623:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205104.989624:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205104.989627:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205104.989629:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205104.989631:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788205104.989634:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989636:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205104.989638:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989640:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205104.989642:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788205104.989645:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989647:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205104.989649:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205104.989656:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205104.989658:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.989660:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205104.989665:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205104.989669:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.989672:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205104.989675:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788205104.989678:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205104.989682:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205104.989683:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.989685:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205104.989686:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205104.989688:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205104.989692:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788205104.989696:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205104.989702:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205104.989710:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205104.989713:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205104.989717:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.989721:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205104.989727:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.989728:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205104.989732:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788205104.989733:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.989739:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:3.0:1788205104.989740:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205104.989741:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205104.989745:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205104.989752:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205104.989753:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205104.989755:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205104.989758:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26791 02000000:00000001:3.0:1788205104.989760:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205104.989763:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1788205104.989765:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205104.989766:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205104.989771:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00100000:3.0:1788205104.989774:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543511424:192.168.201.118@tcp:250:kworker.0 00000100:00080000:0.0:1788205104.989776:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205104.989782:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1788205104.989783:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.989786:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788205104.989787:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1788205104.989788:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:3.0:1788205104.989791:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.989794:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:3.0:1788205104.989795:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1788205104.989799:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205104.989801:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 02000000:00000010:3.0:1788205104.989802:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 00000020:00000001:0.0:1788205104.989803:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205104.989807:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 02000000:00000001:3.0:1788205104.989807:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205104.989812:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788205104.989815:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205104.989818:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:0.0:1788205104.989822:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000010:3.0:1788205104.989824:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:0.0:1788205104.989826:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205104.989830:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:3.0:1788205104.989832:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000001:0.0:1788205104.989833:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205104.989838:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788205104.989841:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000200:3.0:1788205104.989842:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543511424, portal 25 00000100:00000001:0.0:1788205104.989845:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.989847:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788205104.989849:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205104.989852:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904121216 : -109419805430400 : ffff9c7bb75dad80) 02000000:00000001:0.0:1788205104.989854:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205104.989858:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.989860:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:3.0:1788205104.989863:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788205104.989864:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205104.989870:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.989875:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205104.989881:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000100:00000001:3.0:1788205104.989882:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:0.0:1788205104.989886:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000100:00000040:3.0:1788205104.989889:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000001:0.0:1788205104.989891:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205104.989893:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205104.989895:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734a18. 00000100:00000001:0.0:1788205104.989896:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205104.989899:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:3.0:1788205104.989901:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543511424, offset 0 02000000:00000001:0.0:1788205104.989902:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205104.989910:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788205104.989915:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205104.989915:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00100000:0.0:1788205104.989923:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205104.989932:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205104.989942:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205104.989944:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205104.989948:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000040:0.0:1788205104.989952:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205104.989957:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788205104.989979:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205104.989983:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.989986:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205104.989987:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205104.989989:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205104.989993:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205104.989995:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00080000:0.0:1788205104.989999:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26791 00000400:00000200:3.0:1788205104.990004:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788205104.990006:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990010:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:0.0:1788205104.990013:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205104.990018:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205104.990021:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471344:471344:256:4294967295] 192.168.201.118@tcp LPNI seq info [471203:471203:8:4294967295] 00000100:00000040:0.0:1788205104.990024:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990030:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205104.990033:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:0.0:1788205104.990035:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990039:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205104.990042:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205104.990044:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205104.990048:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205104.990053:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000400:00000200:3.0:1788205104.990053:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000020:00000001:0.0:1788205104.990058:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788205104.990062:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:3.0:1788205104.990065:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000040:0.0:1788205104.990068:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990071:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205104.990073:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000010:3.0:1788205104.990074:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000100:00000001:0.0:1788205104.990075:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205104.990081:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788205104.990085:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990116:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205104.990119:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205104.990122:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205104.990124:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.990127:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205104.990129:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788205104.990131:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.990133:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205104.990137:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788205104.990140:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205104.990142:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205104.990143:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.990145:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205104.990147:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205104.990148:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205104.990152:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788205104.990156:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205104.990161:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205104.990168:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205104.990170:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205104.990174:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205104.990182:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205104.990184:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205104.990188:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205104.990189:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205104.990191:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205104.990194:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205104.990197:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26791 00000400:00000200:0.0:1788205104.990201:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205104.990204:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205104.990207:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205104.990210:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990213:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205104.990215:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990217:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205104.990219:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205104.990220:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788205104.990222:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205104.990224:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205104.990226:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205104.990228:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788205104.990236:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990239:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205104.990241:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990243:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205104.990246:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788205104.990248:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205104.990251:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205104.990253:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205104.990257:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205104.990259:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.990261:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205104.990263:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205104.990265:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.990268:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205104.990271:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788205104.990274:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205104.990277:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205104.990279:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.990281:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205104.990283:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.0:1788205104.990299:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205104.990308:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205104.990373:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205104.990380:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205104.990383:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321465095 00000100:00000010:3.0:1788205104.990389:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc4e0. 00000100:00000001:3.0:1788205104.990397:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205104.990404:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321465095 02000000:00000001:0.0:1788205104.990407:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205104.990409:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc240. 00000100:00000001:3.0:1788205104.990414:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000010:0.0:1788205104.990415:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:3.0:1788205104.990418:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205104.990421:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.990423:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205104.990429:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205104.990429:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205104.990438:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205104.990442:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205104.990448:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.990453:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1788205104.990456:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.990457:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205104.990462:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205104.990465:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205104.990470:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205104.990473:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205104.990477:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511552:192.168.201.118@tcp:38:kworker.0 00000100:00000040:0.0:1788205104.990481:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26801) 00000100:00000001:3.0:1788205104.990483:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205104.990487:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205104.990489:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205104.990491:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205104.990495:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205104.990498:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205104.990502:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205104.990506:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb778f0. 00000400:00000010:3.0:1788205104.990511:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734880. 00000100:00000200:3.0:1788205104.990515:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543511552, portal 10 00000100:00000001:3.0:1788205104.990517:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205104.990518:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904110464 : -109419805441152 : ffff9c7bb75d8380) 00000100:00000040:3.0:1788205104.990525:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.990532:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205104.990534:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205104.990537:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734cc0. 00000100:00000200:3.0:1788205104.990540:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543511552, offset 0 00000400:00000200:3.0:1788205104.990544:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205104.990548:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990552:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205104.990555:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205104.990558:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205104.990560:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205104.990563:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205104.990566:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471345:471345:255:4294967295] 192.168.201.118@tcp LPNI seq info [471204:471204:7:4294967295] 00000400:00000200:3.0:1788205104.990570:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990576:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205104.990581:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205104.990585:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdd00. 00000800:00000200:3.0:1788205104.990590:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205104.990593:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205104.990596:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205104.990601:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205104.990604:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205104.990606:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321463828 00000100:00000001:3.0:1788205104.990608:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321465095 00000100:00000001:3.0:1788205104.990610:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321463828 00000100:00000001:3.0:1788205104.990612:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321465095 00000100:00000001:3.0:1788205104.990614:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205104.990616:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.990619:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.990621:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205104.990631:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205104.990633:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788205104.990635:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205104.990637:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205104.990638:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205104.990642:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.990650:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205104.990651:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205104.990653:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205104.990655:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205104.990656:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205104.990659:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511680:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205104.990663:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205104.990665:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205104.990667:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205104.990669:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205104.990672:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205104.990675:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205104.990678:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205104.990681:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77478. 00000400:00000010:3.0:1788205104.990684:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734dd0. 00000100:00000200:3.0:1788205104.990687:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543511680, portal 10 00000100:00000001:3.0:1788205104.990689:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205104.990690:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904122112 : -109419805429504 : ffff9c7bb75db100) 00000100:00000040:3.0:1788205104.990696:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.990702:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205104.990704:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205104.990709:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987347f8. 00000100:00000200:3.0:1788205104.990711:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543511680, offset 0 00000400:00000200:3.0:1788205104.990716:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205104.990720:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990723:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205104.990726:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205104.990729:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205104.990731:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205104.990734:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205104.990737:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471346:471346:254:4294967295] 192.168.201.118@tcp LPNI seq info [471205:471205:6:4294967295] 00000400:00000200:3.0:1788205104.990741:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990746:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205104.990749:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205104.990751:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdb00. 00000800:00000200:3.0:1788205104.990754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205104.990757:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205104.990760:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205104.990762:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205104.990765:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205104.990768:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321463829 00000100:00000001:3.0:1788205104.990770:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321465095 00000100:00000001:3.0:1788205104.990772:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321463829 00000100:00000001:3.0:1788205104.990774:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321465095 00000100:00000001:3.0:1788205104.990776:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205104.990778:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.990780:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205104.990784:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.990790:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205104.990791:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205104.990794:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205104.990796:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205104.990797:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205104.990800:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511808:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205104.990803:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205104.990805:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205104.990807:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205104.990809:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205104.990812:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205104.990814:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205104.990818:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205104.990820:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77e18. 00000400:00000010:3.0:1788205104.990825:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987342a8. 00000100:00000200:3.0:1788205104.990828:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543511808, portal 4 00000100:00000001:3.0:1788205104.990830:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205104.990831:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904109568 : -109419805442048 : ffff9c7bb75d8000) 00000100:00000040:3.0:1788205104.990836:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.990843:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205104.990845:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205104.990848:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987346e8. 00000100:00000200:3.0:1788205104.990851:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543511808, offset 0 00000400:00000200:3.0:1788205104.990855:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205104.990859:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990862:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205104.990864:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205104.990867:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205104.990869:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205104.990871:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205104.990874:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471347:471347:253:4294967295] 192.168.201.118@tcp LPNI seq info [471206:471206:5:4294967295] 00000400:00000200:3.0:1788205104.990878:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990884:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205104.990888:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205104.990891:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205104.990894:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205104.990897:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205104.990898:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205104.990901:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205104.990903:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205104.990904:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321463838 00000100:00000001:3.0:1788205104.990906:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321465095 00000100:00000001:3.0:1788205104.990907:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321463838 00000100:00000001:3.0:1788205104.990909:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321465095 00000100:00000001:3.0:1788205104.990910:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205104.990911:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.990913:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205104.990915:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.990923:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205104.990924:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205104.990926:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205104.990927:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205104.990928:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205104.990930:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511936:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205104.990932:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205104.990933:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205104.990934:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205104.990935:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205104.990937:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205104.990939:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205104.990940:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205104.990942:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77738. 00000400:00000010:3.0:1788205104.990944:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987345d8. 00000100:00000200:3.0:1788205104.990946:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543511936, portal 4 00000100:00000001:3.0:1788205104.990947:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205104.990948:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904111360 : -109419805440256 : ffff9c7bb75d8700) 00000100:00000040:3.0:1788205104.990952:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205104.990956:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205104.990957:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205104.990959:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734110. 00000100:00000200:3.0:1788205104.990960:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543511936, offset 0 00000400:00000200:3.0:1788205104.990962:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205104.990976:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990978:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205104.990980:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205104.990981:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205104.990983:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205104.990984:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205104.990987:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471348:471348:252:4294967295] 192.168.201.118@tcp LPNI seq info [471207:471207:4:4294967295] 00000400:00000200:3.0:1788205104.990989:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205104.990993:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205104.990996:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205104.990999:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd900. 00000800:00000200:3.0:1788205104.991001:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205104.991004:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205104.991006:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205104.991007:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205104.991009:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205104.991011:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321463861 00000100:00000001:3.0:1788205104.991012:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321465095 00000100:00000001:3.0:1788205104.991014:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321463861 00000100:00000001:3.0:1788205104.991015:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321465095 00000100:00000001:3.0:1788205104.991016:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205104.991017:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.991019:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.991020:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205104.991022:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205104.991023:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788205104.991024:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205104.991025:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205104.991026:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205104.991027:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205105.820261:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205105.820273:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205105.820275:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205105.820282:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205105.820287:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205105.820292:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205105.820294:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205105.820299:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205105.820302:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205105.820460:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205105.820464:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205105.820470:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205105.820471:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205105.820474:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205105.820475:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205105.820479:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205105.820481:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205105.820903:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205105.820909:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205105.820913:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205105.821327:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205105.821331:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205105.821333:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205105.821335:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205105.821337:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205105.821339:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205105.821342:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205105.821344:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205105.821346:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205106.332248:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321465095, at 4321465152 00000100:00000040:3.0:1788205106.332249:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321465095, at 4321465152 00000100:00000010:2.0:1788205106.332270:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc4e0. 00000100:00000010:3.0:1788205106.332274:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc240. 00000100:00000001:2.0:1788205106.844217:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205106.844225:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205106.844226:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205106.844232:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205106.844235:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205106.844238:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205106.844241:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205106.844244:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205106.844246:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205106.844249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205106.844255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205106.844261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205106.844263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205106.844267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205106.844268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205106.844272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205106.844273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205106.844554:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205106.844686:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205106.844690:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205106.844720:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205106.844723:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205106.844724:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205106.844726:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205106.844727:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205106.844729:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205106.844731:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205106.844732:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205106.844734:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205107.868244:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205107.868254:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205107.868257:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205107.868260:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205107.868262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205107.868264:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205107.868272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205107.868272:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205107.868273:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205107.868282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205107.868284:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205107.868286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205107.868292:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205107.868293:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205107.868295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205107.868300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205107.868300:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205107.868301:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205107.868309:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205107.868311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205107.868312:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205107.868315:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205107.868315:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205107.868317:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205107.868324:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205107.868325:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205107.868489:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205107.868494:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205107.868498:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788205108.060365:0:256121:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788205108.060409:0:256121:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788205108.060418:0:256121:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205108.060422:0:256121:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788205108.060427:0:256121:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205108.060431:0:256121:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788205108.060436:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205108.060440:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000400:00000200:2.0:1788205108.060445:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205108.060450:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205108.060456:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734a18 00000400:00000010:2.0:1788205108.060459:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734a18. 00000100:00000001:2.0:1788205108.060464:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205108.060473:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205108.060539:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205108.060544:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205108.060548:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788205108.060566:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205108.060571:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205108.062698:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205108] req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205108.062722:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205108.062724:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205108.062730:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205108.062736:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205108.062739:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205108.062743:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205108.062747:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205108.062755:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205108.062762:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205108.062764:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205108.062766:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205108.062768:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205108.062772:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205108.062778:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205108.062782:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 2 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205108.062794:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205108.062797:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205108.062799:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205108.062800:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205108.062803:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205108.062805:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205108.062809:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205108.062814:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205108.062816:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205108.062820:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 2 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205108.063656:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543511424:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205108.063663:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205108.063666:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205108.063671:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dad80 x1875055543511424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205129 ref 2 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205108.063678:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205108.063680:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205108.063682:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205108.063684:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205108.063686:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205108.063688:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.064658:0:256121:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205108.064663:0:256121:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205108.064668:0:256121:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:2.0:1788205108.064673:0:256121:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788205108.064676:0:256121:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:2.0:1788205108.064678:0:256121:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:2.0:1788205108.064681:0:256121:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:2.0:1788205108.064683:0:256121:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29800. 02000000:00000001:2.0:1788205108.064687:0:256121:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788205108.064689:0:256121:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1788205108.064690:0:256121:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dad80. 00000100:00000001:2.0:1788205108.064694:0:256121:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788205108.064696:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205108.064698:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205108.064713:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205108.064716:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdd00. 00000400:00000200:2.0:1788205108.064720:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205108.064725:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205108.064731:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734cc0 00000400:00000010:2.0:1788205108.064734:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734cc0. 00000100:00000001:2.0:1788205108.064737:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205108.064742:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.064767:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205108.064771:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.064778:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.064779:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205108.064784:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205108.064787:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdb00. 00000400:00000200:2.0:1788205108.064789:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205108.064793:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205108.064796:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987347f8 00000400:00000010:2.0:1788205108.064799:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987347f8. 00000100:00000001:2.0:1788205108.064801:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205108.064806:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.064816:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205108.064820:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.064839:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205108.064841:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.064847:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205108.064849:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205108.064854:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000800:00000100:2.0:1788205108.064855:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205108.064860:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:2.0:1788205108.064866:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:1.0:1788205108.064874:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205108] req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205108.064875:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205108.064880:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987346e8 00000400:00000010:2.0:1788205108.064882:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987346e8. 00000100:00000001:2.0:1788205108.064884:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205108.064890:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.064928:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205108.064935:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.064942:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.064943:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205108.064948:0:256121:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205108.064950:0:256121:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd900. 00000400:00000200:2.0:1788205108.064953:0:256121:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205108.064956:0:256121:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000001:1.0:1788205108.065064:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205108.065068:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205108.065072:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77478. 00000400:00000200:1.0:1788205108.065079:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734dd0 00000400:00000010:1.0:1788205108.065082:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734dd0. 00000100:00000001:1.0:1788205108.065086:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205108.065133:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205108.065155:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.065163:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205108.065165:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.065169:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205108.065171:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205108.065176:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.065183:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205108.065188:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.065195:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205108.065199:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.065200:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205108.065201:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.065214:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205108.065219:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205108.065225:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205108.065230:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205108.065232:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205108.065236:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205108.065245:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511680:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205108.065250:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205108.065254:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205108] req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205108.066979:0:256121:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734110 00000400:00000010:2.0:1788205108.066985:0:256121:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734110. 00000400:00000001:1.0:1788205108.067028:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205108.067031:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205108.067034:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77e18. 00000400:00000200:1.0:1788205108.067037:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987342a8 00000400:00000010:1.0:1788205108.067039:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987342a8. 00000100:00000001:1.0:1788205108.067051:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205108.067060:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205108.067071:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067079:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205108.067082:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067111:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205108.067500:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205108.067507:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067515:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205108.067521:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067533:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205108.067536:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067538:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205108.067539:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067542:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205108.067545:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205108.067549:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205108.067554:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205108.067557:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205108.067561:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205108.067568:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511808:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205108.067574:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205108.067578:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205108] req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205108.067586:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205108.067587:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205108.067589:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb778f0. 00000400:00000200:1.0:1788205108.067592:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734880 00000400:00000010:1.0:1788205108.067594:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734880. 00000100:00000001:1.0:1788205108.067596:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205108.067599:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205108.067606:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067611:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205108.067612:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067614:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205108.067615:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205108.067618:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067624:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205108.067627:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067632:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205108.067646:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067648:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205108.067650:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067651:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205108.067653:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205108.067656:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205108.067660:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205108.067662:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205108.067666:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205108.067674:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511552:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205108.067678:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205108.067681:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205108.067684:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db100 x1875055543511680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067688:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205108.067690:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205108.067693:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:1.0:1788205108.067697:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205108.067698:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205108.067700:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205108.067702:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205108.067704:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29000. 02000000:00000001:1.0:1788205108.067706:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205108.067707:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205108.067709:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db100. 00000100:00000001:1.0:1788205108.067711:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205108.067712:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067713:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205108.067715:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8000 x1875055543511808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067719:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205108.067719:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205108.067720:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:1.0:1788205108.067722:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205108.067722:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205108.067723:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205108.067724:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205108.067724:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b400. 02000000:00000001:1.0:1788205108.067726:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205108.067726:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205108.067727:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8000. 00000100:00000001:1.0:1788205108.067728:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205108.067729:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067730:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205108.067731:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8380 x1875055543511552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.067734:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205108.067735:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205108.067736:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205108.067737:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205108.067737:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205108.067738:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205108.067739:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205108.067740:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:1.0:1788205108.067741:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205108.067742:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205108.067743:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8380. 00000100:00000001:1.0:1788205108.067744:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205108.067745:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.067746:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205108.067748:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205108.067749:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205108.067750:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205108.067751:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.068329:0:256121:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205108.068335:0:256121:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.068356:0:256121:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205108.068359:0:256121:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205129 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205108.068365:0:256121:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.068367:0:256121:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205108.068407:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205108.068409:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205108.068410:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205108.068414:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205104/real 1788205108] req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205108.068421:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205108.068422:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205108.068423:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77738. 00000400:00000200:1.0:1788205108.068425:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987345d8 00000400:00000010:1.0:1788205108.068426:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987345d8. 00000100:00000001:1.0:1788205108.068428:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205108.068430:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205108.068435:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.068439:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205108.068440:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.068441:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205108.068442:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205108.068444:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.068447:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205108.068449:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.068457:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205108.068458:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.068459:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205108.068460:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.068461:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205108.068462:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205108.068464:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205108.068466:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205108.068467:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205108.068469:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205108.068473:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543511936:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205108.068475:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.068477:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205108.068478:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8700 x1875055543511936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205129 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205108.068482:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205108.068483:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205108.068484:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205108.068486:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205108.068486:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205108.068487:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205108.068488:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205108.068489:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28c00. 02000000:00000001:1.0:1788205108.068491:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205108.068491:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205108.068492:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8700. 00000100:00000001:1.0:1788205108.068494:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205108.068494:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.068496:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205108.068501:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205108.068502:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205108.068504:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205108.068506:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205108.892311:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205108.892320:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205108.892322:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205108.892329:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205108.892333:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205108.892337:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205108.892340:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205108.892343:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205108.892347:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205108.892468:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205108.892472:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205108.892473:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205108.892476:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205108.892478:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205108.892480:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205108.892483:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205108.892485:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205108.892487:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205108.892678:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205108.892683:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205108.892688:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205108.892689:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205108.892693:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205108.892694:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205108.892698:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205108.892699:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205108.892827:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205108.892829:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205108.892832:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205109.916215:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205109.916217:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205109.916228:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205109.916228:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205109.916232:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205109.916233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205109.916242:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205109.916243:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205109.916250:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205109.916251:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205109.916256:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205109.916260:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205109.916262:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205109.916264:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205109.916267:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205109.916271:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205109.916277:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205109.916277:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205109.916388:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205109.916394:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205109.916396:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205109.916401:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205109.916404:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205109.916407:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205109.916409:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205109.916412:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205109.916418:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205109.916419:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205109.916423:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205109.916424:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205109.916428:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205109.916429:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205109.916500:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205109.916503:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205109.916506:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205110.108160:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205110.108173:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205110.108176:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205110.108181:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205110.108187:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205110.108196:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26796 00000400:00000200:0.0:1788205110.108208:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205110.108217:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205110.108221:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205110.108225:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108230:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108233:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108236:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205110.108241:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205110.108243:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205110.108246:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205110.108250:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205110.108253:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205110.108256:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205110.108259:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108261:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108263:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108265:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205110.108268:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205110.108272:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108275:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205110.108279:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205110.108283:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205110.108287:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108290:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205110.108293:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205110.108301:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788205110.108305:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205110.108310:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205110.108320:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108324:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205110.108328:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205110.108334:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205110.108344:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788205110.108354:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205110.108364:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205110.108382:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205110.108387:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205110.108392:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205110.108424:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205110.108428:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205110.108431:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205110.108433:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205110.108435:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205110.108438:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205110.108441:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26796 00000400:00000200:0.0:1788205110.108445:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205110.108448:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205110.108451:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205110.108453:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108455:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108457:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108459:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205110.108462:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205110.108463:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205110.108465:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205110.108467:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205110.108469:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205110.108471:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788205110.108474:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108476:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108478:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108480:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205110.108483:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788205110.108486:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108488:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205110.108490:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205110.108496:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205110.108498:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108501:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205110.108506:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205110.108509:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108511:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205110.108515:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788205110.108517:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205110.108519:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205110.108521:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108522:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205110.108524:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205110.108526:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205110.108530:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788205110.108533:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205110.108538:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205110.108546:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205110.108548:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205110.108552:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205110.108559:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205110.108562:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205110.108566:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205110.108568:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205110.108569:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205110.108572:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205110.108575:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26796 00000400:00000200:0.0:1788205110.108579:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205110.108582:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205110.108584:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205110.108587:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108589:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108591:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108593:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205110.108595:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205110.108597:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788205110.108598:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205110.108600:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205110.108602:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788205110.108604:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788205110.108607:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108609:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108611:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108613:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205110.108616:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788205110.108618:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108621:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205110.108622:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205110.108625:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205110.108627:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108630:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205110.108632:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205110.108634:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108635:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205110.108638:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788205110.108641:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205110.108643:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205110.108645:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108647:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205110.108648:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205110.108650:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205110.108654:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788205110.108658:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205110.108662:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205110.108669:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205110.108671:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205110.108675:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205110.108682:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205110.108685:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205110.108688:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205110.108690:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205110.108692:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205110.108694:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205110.108698:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26796 00000400:00000200:0.0:1788205110.108702:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205110.108705:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205110.108707:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205110.108710:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108712:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108714:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108716:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205110.108718:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205110.108719:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205110.108721:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205110.108723:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788205110.108725:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788205110.108728:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788205110.108731:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108733:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108734:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108737:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205110.108739:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788205110.108742:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108744:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205110.108746:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205110.108749:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205110.108751:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108753:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205110.108755:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788205110.108758:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108759:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205110.108763:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788205110.108765:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205110.108767:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205110.108768:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108770:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205110.108771:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205110.108773:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205110.108776:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788205110.108779:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205110.108783:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205110.108790:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205110.108792:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205110.108795:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205110.108802:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205110.108805:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205110.108809:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205110.108810:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205110.108812:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205110.108815:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205110.108817:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26796 00000400:00000200:0.0:1788205110.108820:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205110.108823:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205110.108825:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205110.108827:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108830:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108831:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108833:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205110.108835:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205110.108837:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788205110.108838:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205110.108840:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205110.108842:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205110.108844:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788205110.108847:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108849:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205110.108851:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108853:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205110.108855:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788205110.108858:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205110.108860:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205110.108861:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205110.108864:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205110.108873:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108875:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205110.108888:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205110.108890:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108892:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205110.108895:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788205110.108897:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205110.108899:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205110.108900:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205110.108902:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205110.108903:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205110.108905:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205110.108908:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788205110.108911:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205110.108916:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205110.108922:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205110.108925:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205110.108929:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205110.108936:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205110.108943:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26806) 00000100:00000001:1.0:1788205110.108943:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205110.108951:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205110.108956:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205110.108964:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.108974:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205110.108977:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.108983:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205110.108985:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205110.108988:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205110.108992:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543512064:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205110.108996:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205110.109003:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205110.109006:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.109009:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205110.109017:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bbc00. 02000000:00000001:1.0:1788205110.109019:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205110.109026:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205110.109032:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205110.109044:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205110.109056:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543512064, portal 25 00000100:00000001:1.0:1788205110.109059:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205110.109060:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904118528 : -109419805433088 : ffff9c7bb75da300) 00000100:00000040:1.0:1788205110.109066:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109074:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205110.109077:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205110.109080:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205110.109085:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543512064, offset 0 00000400:00000200:1.0:1788205110.109123:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205110.109134:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.109141:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205110.109148:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205110.109159:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205110.109164:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205110.109167:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205110.109172:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471349:471349:256:4294967295] 192.168.201.118@tcp LPNI seq info [471208:471208:8:4294967295] 00000400:00000200:1.0:1788205110.109177:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.109189:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205110.109196:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.109203:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205110.109213:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205110.109217:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205110.109224:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205110.109227:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205110.109232:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.109235:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205110.109242:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205110.109249:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205110.109257:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205110.109260:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205110.109265:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205110.109276:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109288:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.109292:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109299:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.109301:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205110.109310:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205110.109315:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205110.109319:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321470214 00000100:00000010:1.0:1788205110.109326:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647720. 00000100:00000001:1.0:1788205110.109335:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205110.109343:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321470214 00000100:00000010:1.0:1788205110.109346:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647600. 00000100:00000001:1.0:1788205110.109350:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205110.109354:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205110.109356:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.109360:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205110.109368:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205110/real 1788205110] req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205110.109379:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205110.109382:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205110.109385:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205110.109388:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205110.109390:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205110.109393:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205110.109398:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205110.109407:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109415:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205110.109417:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.109420:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.109423:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205110.109428:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109435:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205110.109441:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109447:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205110.109451:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.109453:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.109454:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.109457:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205110.109459:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205110.109463:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205110.109467:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205110.109469:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205110.109474:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205110.109482:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543512064:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205110.109488:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205110.109491:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109498:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205110.109499:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.109501:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205110.109504:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205110.109506:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205110.109510:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512192:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205110.109514:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205110.109516:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205110.109517:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.109519:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205110.109523:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb800. 02000000:00000001:1.0:1788205110.109526:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205110.109531:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205110.109534:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205110.109537:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205110.109541:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543512192, portal 10 00000100:00000001:1.0:1788205110.109544:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205110.109546:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114048 : -109419805437568 : ffff9c7bb75d9180) 00000100:00000040:1.0:1788205110.109554:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109560:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205110.109563:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205110.109566:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205110.109568:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543512192, offset 0 00000400:00000200:1.0:1788205110.109572:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205110.109577:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.109581:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205110.109584:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205110.109587:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205110.109590:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205110.109592:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205110.109595:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471350:471350:256:4294967295] 192.168.201.118@tcp LPNI seq info [471209:471209:8:4294967295] 00000400:00000200:1.0:1788205110.109599:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.109605:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205110.109610:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.109615:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205110.109619:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205110.109621:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205110.109626:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205110.109629:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205110.109633:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.109635:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205110.109638:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205110.109642:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205110.109645:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205110.109648:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205110.109651:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205110.109654:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109666:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.109669:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109675:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.109676:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205110.109680:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205110.109684:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205110.109687:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205110.109689:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321468947 00000100:00000001:1.0:1788205110.109692:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321470214 00000100:00000001:1.0:1788205110.109694:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321468947 00000100:00000001:1.0:1788205110.109696:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321470214 00000100:00000001:1.0:1788205110.109698:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205110.109700:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.109702:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205110.109706:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205110/real 1788205110] req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205110.109713:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205110.109714:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205110.109716:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205110.109718:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205110.109720:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205110.109722:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205110.109725:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205110.109733:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109740:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205110.109743:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.109745:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.109746:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205110.109749:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109756:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205110.109759:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109766:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205110.109769:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.109770:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.109772:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.109774:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205110.109777:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205110.109779:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205110.109783:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205110.109785:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205110.109789:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205110.109796:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512192:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205110.109800:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205110.109803:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109809:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205110.109810:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.109812:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205110.109814:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205110.109816:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205110.109819:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512320:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205110.109822:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205110.109824:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205110.109826:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.109828:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205110.109833:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205110.109835:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205110.109838:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205110.109840:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205110.109843:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205110.109846:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543512320, portal 10 00000100:00000001:1.0:1788205110.109848:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205110.109849:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904123904 : -109419805427712 : ffff9c7bb75db800) 00000100:00000040:1.0:1788205110.109855:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109861:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205110.109863:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205110.109865:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205110.109867:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543512320, offset 0 00000400:00000200:1.0:1788205110.109871:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205110.109875:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.109895:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205110.109898:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205110.109900:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205110.109902:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205110.109905:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205110.109908:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471351:471351:256:4294967295] 192.168.201.118@tcp LPNI seq info [471210:471210:8:4294967295] 00000400:00000200:1.0:1788205110.109912:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.109919:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205110.109923:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.109926:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205110.109930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205110.109934:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205110.109938:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205110.109942:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205110.109945:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.109947:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205110.109950:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205110.109954:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205110.109957:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205110.109958:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205110.109961:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205110.109965:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109973:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.109975:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.109981:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.109983:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205110.109986:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205110.109989:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205110.109992:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205110.109994:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321468948 00000100:00000001:1.0:1788205110.109997:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321470214 00000100:00000001:1.0:1788205110.109999:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321468948 00000100:00000001:1.0:1788205110.110001:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321470214 00000100:00000001:1.0:1788205110.110003:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205110.110005:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.110007:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205110.110011:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205110/real 1788205110] req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205110.110018:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205110.110020:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205110.110022:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205110.110024:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205110.110025:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205110.110028:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205110.110030:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205110.110040:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110046:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205110.110048:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110049:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.110051:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205110.110054:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110061:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205110.110065:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110071:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205110.110073:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110075:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.110076:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110078:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205110.110081:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205110.110083:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205110.110086:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205110.110088:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205110.110128:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205110.110142:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512320:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205110.110146:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205110.110149:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110156:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205110.110158:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.110160:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205110.110162:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205110.110163:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205110.110166:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512448:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205110.110169:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205110.110172:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205110.110173:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.110175:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205110.110179:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205110.110181:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205110.110185:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205110.110188:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205110.110191:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205110.110194:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543512448, portal 4 00000100:00000001:1.0:1788205110.110197:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205110.110198:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904113152 : -109419805438464 : ffff9c7bb75d8e00) 00000100:00000040:1.0:1788205110.110204:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110210:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205110.110212:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205110.110214:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205110.110216:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543512448, offset 0 00000400:00000200:1.0:1788205110.110221:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205110.110226:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.110230:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205110.110232:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205110.110235:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205110.110238:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205110.110240:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205110.110242:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471352:471352:256:4294967295] 192.168.201.118@tcp LPNI seq info [471211:471211:8:4294967295] 00000400:00000200:1.0:1788205110.110246:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.110251:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205110.110256:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.110259:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205110.110263:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205110.110265:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205110.110268:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205110.110271:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205110.110274:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.110276:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205110.110278:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205110.110281:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205110.110284:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205110.110286:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205110.110288:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205110.110291:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110298:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.110301:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110306:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.110307:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205110.110310:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205110.110314:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205110.110316:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205110.110318:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321468958 00000100:00000001:1.0:1788205110.110321:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321470214 00000100:00000001:1.0:1788205110.110322:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321468958 00000100:00000001:1.0:1788205110.110324:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321470214 00000100:00000001:1.0:1788205110.110326:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205110.110328:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.110329:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205110.110332:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205110/real 1788205110] req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205110.110339:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205110.110340:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205110.110342:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205110.110344:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205110.110345:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205110.110347:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205110.110350:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205110.110357:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110362:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205110.110364:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110366:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.110367:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205110.110370:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110376:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205110.110379:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110385:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205110.110388:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110390:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.110391:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110393:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205110.110394:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205110.110397:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205110.110401:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205110.110403:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205110.110406:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205110.110414:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512448:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205110.110418:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205110.110422:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110429:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205110.110430:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.110433:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205110.110435:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205110.110437:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205110.110440:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512576:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205110.110443:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205110.110445:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205110.110447:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205110.110449:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205110.110454:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205110.110455:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205110.110458:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205110.110460:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205110.110463:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205110.110466:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543512576, portal 4 00000100:00000001:1.0:1788205110.110468:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205110.110469:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114944 : -109419805436672 : ffff9c7bb75d9500) 00000100:00000040:1.0:1788205110.110474:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110480:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205110.110482:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205110.110485:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205110.110487:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543512576, offset 0 00000400:00000200:1.0:1788205110.110491:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205110.110495:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.110498:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205110.110500:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205110.110502:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205110.110505:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205110.110507:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205110.110510:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471353:471353:256:4294967295] 192.168.201.118@tcp LPNI seq info [471212:471212:8:4294967295] 00000400:00000200:1.0:1788205110.110514:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205110.110519:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205110.110523:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.110526:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205110.110529:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205110.110532:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205110.110536:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205110.110538:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205110.110542:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205110.110544:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205110.110546:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205110.110550:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205110.110553:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205110.110554:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205110.110557:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205110.110560:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110572:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.110575:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205135 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110582:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.110583:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205110.110586:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205110.110589:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205110.110593:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205110.110595:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321468981 00000100:00000001:1.0:1788205110.110597:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321470214 00000100:00000001:1.0:1788205110.110599:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321468981 00000100:00000001:1.0:1788205110.110601:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321470214 00000100:00000001:1.0:1788205110.110603:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205110.110605:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.110607:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205110.110611:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205110/real 1788205110] req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205110.110618:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205110.110620:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205110.110622:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205110.110624:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205110.110626:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205110.110628:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205110.110630:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205110.110637:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110643:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205110.110645:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110647:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.110648:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205110.110652:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110658:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205110.110662:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110669:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205110.110671:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110674:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205110.110675:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110678:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205110.110680:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205110.110683:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205110.110687:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205110.110689:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205110.110692:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205110.110699:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512576:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205110.110703:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110705:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.110708:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da300 x1875055543512064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110715:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205110.110718:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205110.110721:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:1.0:1788205110.110724:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205110.110726:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205110.110729:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205110.110731:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205110.110734:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2ac00. 02000000:00000001:1.0:1788205110.110739:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205110.110741:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205110.110743:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da300. 00000100:00000001:1.0:1788205110.110747:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205110.110748:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110749:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.110752:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9180 x1875055543512192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110759:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205110.110760:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205110.110762:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:1.0:1788205110.110763:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205110.110764:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205110.110765:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205110.110767:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205110.110768:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a800. 02000000:00000001:1.0:1788205110.110770:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205110.110771:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205110.110772:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9180. 00000100:00000001:1.0:1788205110.110774:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205110.110775:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110777:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.110780:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db800 x1875055543512320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110786:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205110.110787:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205110.110788:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205110.110790:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205110.110791:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205110.110792:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205110.110793:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205110.110794:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:1.0:1788205110.110796:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205110.110797:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205110.110798:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db800. 00000100:00000001:1.0:1788205110.110800:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205110.110802:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110803:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.110806:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8e00 x1875055543512448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110811:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205110.110812:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205110.110814:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205110.110815:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205110.110816:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205110.110817:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205110.110819:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205110.110820:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:1.0:1788205110.110823:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205110.110824:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205110.110825:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8e00. 00000100:00000001:1.0:1788205110.110827:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205110.110828:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110829:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205110.110832:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9500 x1875055543512576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205135 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205110.110837:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205110.110838:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205110.110839:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205110.110841:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205110.110842:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205110.110843:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205110.110845:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205110.110846:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29400. 02000000:00000001:1.0:1788205110.110848:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205110.110849:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205110.110851:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9500. 00000100:00000001:1.0:1788205110.110853:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205110.110854:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110857:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205110.110868:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205110.110870:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205110.110873:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205110.110874:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205110.940234:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205110.940243:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205110.940245:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205110.940252:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205110.940256:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205110.940261:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205110.940265:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205110.940269:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205110.940272:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205110.940500:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205110.940506:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205110.940507:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205110.940510:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205110.940513:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205110.940515:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205110.940517:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205110.940519:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205110.940521:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205110.941945:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205110.941951:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205110.941958:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205110.941959:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205110.941963:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205110.941964:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205110.941968:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205110.941969:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205110.942075:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205110.942080:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205110.942083:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205111.452239:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321470214, at 4321470272 00000100:00000040:3.0:1788205111.452257:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321470214, at 4321470272 00000100:00000010:2.0:1788205111.452263:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647720. 00000100:00000010:3.0:1788205111.452266:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00010000:00000001:0.0:1788205111.964256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205111.964266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205111.964274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205111.964275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205111.964279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205111.964281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205111.964285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205111.964286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205111.964941:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205111.964946:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205111.964949:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205111.964953:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205111.964958:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205111.964962:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205111.964965:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205111.964970:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205111.964973:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205111.965190:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205111.965195:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205111.965199:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205111.965227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205111.965233:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205111.965234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205111.965237:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205111.965239:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205111.965242:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205111.965244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205111.965246:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205111.965249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205112.988230:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205112.988230:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205112.988235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205112.988241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205112.988242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205112.988242:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205112.988247:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205112.988247:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205112.988250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205112.988255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205112.988256:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788205112.988257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205112.988259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205112.988263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205112.988264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205112.988265:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205112.988269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205112.988271:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205112.988271:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205112.988272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205112.988280:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205112.988280:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205112.988286:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205112.988286:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205112.988291:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205112.988291:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205112.988479:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205112.988483:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205112.988487:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205114.012225:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205114.012232:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205114.012234:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205114.012241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205114.012242:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205114.012246:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205114.012248:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205114.012251:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205114.012253:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205114.012256:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205114.012264:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205114.012267:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205114.012271:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012275:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205114.012277:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012280:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012282:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205114.012288:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205114.012289:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205114.012290:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012292:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205114.012293:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012295:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012297:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205114.012302:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205114.012304:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205114.012305:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012306:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205114.012307:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012309:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205114.012311:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205114.012431:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205114.012436:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205114.012440:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205114.012463:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205114.012468:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205114.012469:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205114.012472:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205114.012474:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205114.012477:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205114.012479:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205114.012481:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205114.012483:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205115.036225:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205115.036225:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205115.036232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205115.036235:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205115.036236:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205115.036238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.036239:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205115.036240:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205115.036248:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205115.036248:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205115.036249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205115.036254:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205115.036255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205115.036255:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205115.036261:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.036261:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205115.036262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205115.036266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205115.036269:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205115.036270:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205115.036271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205115.036275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205115.036275:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.036275:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205115.036281:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205115.036283:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205115.036350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205115.036355:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205115.036359:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205115.228267:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205115.228277:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205115.228279:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205115.228282:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205115.228287:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205115.228294:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26801 00000400:00000200:3.0:1788205115.228305:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205115.228313:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205115.228316:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205115.228319:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228321:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205115.228323:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228326:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205115.228329:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205115.228331:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205115.228334:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205115.228337:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205115.228339:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205115.228341:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205115.228343:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228345:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205115.228347:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228349:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205115.228352:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205115.228356:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228359:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205115.228362:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205115.228367:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205115.228370:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228373:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205115.228376:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205115.228389:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788205115.228393:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205115.228397:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205115.228399:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228403:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205115.228406:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205115.228412:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205115.228428:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788205115.228437:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205115.228447:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205115.228459:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205115.228465:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205115.228471:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205115.228497:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205115.228501:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205115.228504:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205115.228506:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205115.228507:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205115.228510:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205115.228513:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26801 00000400:00000200:3.0:1788205115.228516:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205115.228519:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205115.228520:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205115.228522:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228524:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205115.228526:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228527:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205115.228529:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205115.228530:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205115.228531:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205115.228533:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205115.228534:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205115.228535:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205115.228537:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228539:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205115.228541:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228543:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205115.228545:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1788205115.228546:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205115.228552:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205115.228553:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205115.228557:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788205115.228557:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205115.228563:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205115.228568:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205115.228572:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205115.228577:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228581:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205115.228586:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788205115.228587:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788205115.228589:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788205115.228593:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.228596:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205115.228596:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.228600:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1788205115.228602:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205115.228606:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205115.228607:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205115.228609:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205115.228612:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205115.228614:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543512704:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205115.228614:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205115.228617:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205115.228619:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.228622:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1788205115.228624:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205115.228628:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205115.228629:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.228634:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205115.228636:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205115.228639:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788205115.228645:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205115.228647:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205115.228649:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 00000100:00000040:3.0:1788205115.228651:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205115.228653:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228661:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.228665:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205115.228670:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00080000:3.0:1788205115.228674:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205115.228679:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:1.0:1788205115.228680:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205115.228681:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205115.228684:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205115.228689:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205115.228692:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26801 00000400:00000200:3.0:1788205115.228697:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205115.228700:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205115.228703:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205115.228706:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000200:1.0:1788205115.228707:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543512704, portal 25 00000100:00000001:3.0:1788205115.228709:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205115.228712:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1788205115.228713:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205115.228716:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000001:3.0:1788205115.228718:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205115.228722:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205115.228725:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000100:00000040:1.0:1788205115.228726:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205115.228728:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205115.228732:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205115.228736:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205115.228741:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788205115.228744:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205115.228748:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205115.228750:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228753:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1788205115.228756:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000040:3.0:1788205115.228758:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228761:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000200:1.0:1788205115.228763:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543512704, offset 0 00000020:00000040:3.0:1788205115.228771:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:1.0:1788205115.228775:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205115.228779:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228783:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205115.228785:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205115.228801:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205115.228804:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228806:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205115.228809:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205115.228811:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228813:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205115.228817:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788205115.228819:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205115.228821:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205115.228821:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228823:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205115.228824:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205115.228825:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205115.228828:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788205115.228832:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205115.228835:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205115.228845:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205115.228846:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205115.228848:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205115.228854:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205115.228856:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205115.228859:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205115.228860:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205115.228861:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205115.228863:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205115.228865:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26801 00000400:00000200:3.0:1788205115.228867:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205115.228869:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205115.228871:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205115.228872:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228874:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205115.228875:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228877:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205115.228884:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205115.228885:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205115.228886:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205115.228891:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205115.228894:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205115.228896:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205115.228899:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228901:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205115.228903:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228905:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205115.228908:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205115.228910:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228913:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205115.228915:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205115.228918:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205115.228920:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228921:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205115.228923:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205115.228925:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228928:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205115.228930:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205115.228932:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205115.228933:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205115.228934:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.228936:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205115.228937:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205115.228939:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205115.228943:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:3.0:1788205115.228946:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205115.228950:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205115.228956:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205115.228958:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205115.228962:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205115.228969:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205115.228971:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205115.228974:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205115.228976:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205115.228978:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205115.228980:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205115.228983:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26801 00000400:00000200:3.0:1788205115.228986:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205115.228989:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205115.228991:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205115.228993:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228996:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205115.228997:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.228999:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205115.229002:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205115.229003:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205115.229004:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205115.229006:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205115.229008:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205115.229011:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205115.229013:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.229015:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205115.229017:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205115.229020:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205115.229022:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:3.0:1788205115.229028:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000200:1.0:1788205115.229033:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00080000:3.0:1788205115.229034:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205115.229040:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205115.229043:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205115.229044:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1788205115.229047:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205115.229050:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.229051:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:3.0:1788205115.229054:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205115.229057:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000400:00000200:1.0:1788205115.229059:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205115.229064:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000008:00000001:3.0:1788205115.229068:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.229069:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471354:471354:256:4294967295] 192.168.201.118@tcp LPNI seq info [471213:471213:8:4294967295] 00000100:00000001:3.0:1788205115.229076:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.229077:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:3.0:1788205115.229086:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205115.229123:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205115.229127:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205115.229128:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205115.229130:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205115.229131:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205115.229133:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205115.229137:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788205115.229141:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205115.229147:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205115.229154:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205115.229156:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205115.229160:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205115.229167:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205115.229172:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26811) 00000400:00000200:1.0:1788205115.229233:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205115.229242:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.229248:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205115.229256:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205115.229261:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205115.229267:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205115.229271:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205115.229276:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.229279:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205115.229287:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205115.229292:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205115.229298:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205115.229301:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205115.229307:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205115.229311:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229321:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.229325:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229332:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.229334:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205115.229341:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.229345:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205115.229349:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321475334 00000100:00000010:1.0:1788205115.229355:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647600. 00000100:00000001:1.0:1788205115.229362:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205115.229368:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321475334 00000100:00000010:1.0:1788205115.229372:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647720. 00000100:00000001:1.0:1788205115.229375:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205115.229378:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205115.229382:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.229384:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205115.229388:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205115/real 1788205115] req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205115.229405:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205115.229407:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205115.229408:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205115.229411:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205115.229413:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205115.229416:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205115.229420:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205115.229428:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229435:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205115.229436:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229439:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.229442:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205115.229446:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229453:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205115.229458:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229465:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205115.229469:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229470:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.229472:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229474:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205115.229476:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205115.229480:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205115.229484:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205115.229487:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205115.229491:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205115.229501:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543512704:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205115.229506:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229508:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.229511:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543512704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229518:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205115.229521:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205115.229524:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205115.229527:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205115.229529:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205115.229531:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205115.229533:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205115.229536:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205115.229540:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205115.229542:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205115.229544:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205115.229548:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205115.229549:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229551:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229559:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205115.229561:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.229563:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205115.229565:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205115.229567:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205115.229571:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229578:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205115.229579:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.229582:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205115.229584:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205115.229587:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205115.229590:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512832:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205115.229594:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205115.229596:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205115.229598:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.229600:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205115.229603:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205115.229605:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.229609:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205115.229612:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205115.229616:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205115.229619:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543512832, portal 10 00000100:00000001:1.0:1788205115.229621:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205115.229622:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205115.229630:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229636:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205115.229638:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205115.229641:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205115.229643:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543512832, offset 0 00000400:00000200:1.0:1788205115.229647:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205115.229650:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205115.229654:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205115.229657:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205115.229660:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205115.229662:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205115.229664:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205115.229667:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471355:471355:256:4294967295] 192.168.201.118@tcp LPNI seq info [471214:471214:8:4294967295] 00000400:00000200:1.0:1788205115.229672:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205115.229678:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205115.229682:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.229686:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205115.229689:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205115.229692:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205115.229695:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205115.229696:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205115.229699:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.229700:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205115.229702:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205115.229704:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205115.229707:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205115.229708:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205115.229710:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205115.229712:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229717:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.229719:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229723:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.229724:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205115.229726:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205115.229729:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205115.229731:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205115.229733:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321474067 00000100:00000001:1.0:1788205115.229735:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321475334 00000100:00000001:1.0:1788205115.229736:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321474067 00000100:00000001:1.0:1788205115.229738:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321475334 00000100:00000001:1.0:1788205115.229739:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205115.229741:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.229742:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205115.229744:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205115/real 1788205115] req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205115.229749:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205115.229750:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205115.229752:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205115.229753:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205115.229754:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205115.229756:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205115.229757:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205115.229762:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229765:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205115.229766:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229767:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.229767:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205115.229769:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229773:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205115.229775:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229778:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205115.229780:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229781:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.229782:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.229783:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205115.229785:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205115.229786:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205115.229920:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205115.229923:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205115.229927:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205115.229934:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512832:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205115.229938:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205115.229941:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.229946:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205115.229948:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.229949:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205115.229951:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205115.229953:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205115.229955:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512960:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205115.229958:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205115.229961:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205115.229962:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.229964:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205115.229967:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205115.229969:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.229973:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205115.229975:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205115.229979:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205115.229982:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543512960, portal 10 00000100:00000001:1.0:1788205115.229984:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205115.229985:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205115.229992:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230003:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205115.230005:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205115.230008:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205115.230011:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543512960, offset 0 00000400:00000200:1.0:1788205115.230015:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205115.230019:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205115.230023:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205115.230025:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205115.230028:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205115.230030:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205115.230032:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205115.230035:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471356:471356:256:4294967295] 192.168.201.118@tcp LPNI seq info [471215:471215:8:4294967295] 00000400:00000200:1.0:1788205115.230040:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205115.230046:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205115.230050:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.230053:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205115.230057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205115.230060:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205115.230065:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205115.230067:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205115.230071:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.230073:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205115.230075:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205115.230079:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205115.230082:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205115.230084:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205115.230086:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205115.230109:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230115:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.230118:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230122:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.230123:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205115.230126:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205115.230128:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205115.230130:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205115.230131:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321474069 00000100:00000001:1.0:1788205115.230133:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321475334 00000100:00000001:1.0:1788205115.230135:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321474069 00000100:00000001:1.0:1788205115.230137:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321475334 00000100:00000001:1.0:1788205115.230139:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205115.230141:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.230142:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205115.230146:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205115/real 1788205115] req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205115.230154:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205115.230155:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205115.230157:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205115.230159:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205115.230161:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205115.230163:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205115.230166:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205115.230175:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230181:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205115.230182:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230184:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.230185:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205115.230188:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230192:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205115.230194:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230198:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205115.230199:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230200:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.230201:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230203:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205115.230205:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205115.230207:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205115.230209:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205115.230211:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205115.230213:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205115.230218:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543512960:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205115.230220:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205115.230222:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230225:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205115.230226:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.230227:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205115.230229:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205115.230230:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205115.230232:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513088:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205115.230234:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205115.230236:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205115.230237:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.230238:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205115.230240:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205115.230242:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.230245:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205115.230246:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205115.230249:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205115.230251:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543513088, portal 4 00000100:00000001:1.0:1788205115.230252:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205115.230253:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788205115.230257:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230261:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205115.230263:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205115.230264:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205115.230266:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543513088, offset 0 00000400:00000200:1.0:1788205115.230268:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205115.230270:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205115.230272:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205115.230274:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205115.230276:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205115.230277:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205115.230279:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205115.230281:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471357:471357:256:4294967295] 192.168.201.118@tcp LPNI seq info [471216:471216:8:4294967295] 00000400:00000200:1.0:1788205115.230283:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205115.230287:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205115.230290:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.230291:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205115.230294:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205115.230296:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205115.230298:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205115.230300:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205115.230302:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.230304:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205115.230306:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205115.230308:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205115.230310:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205115.230311:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205115.230313:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205115.230314:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230319:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.230321:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230332:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.230334:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205115.230337:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205115.230340:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205115.230343:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205115.230345:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321474078 00000100:00000001:1.0:1788205115.230347:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321475334 00000100:00000001:1.0:1788205115.230350:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321474078 00000100:00000001:1.0:1788205115.230351:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321475334 00000100:00000001:1.0:1788205115.230353:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205115.230355:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.230357:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205115.230361:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205115/real 1788205115] req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205115.230368:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205115.230370:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205115.230371:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205115.230374:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205115.230375:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205115.230377:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205115.230380:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205115.230388:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230394:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205115.230396:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230397:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.230399:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205115.230403:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230410:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205115.230414:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230421:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205115.230424:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230425:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.230427:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230429:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205115.230431:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205115.230433:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205115.230437:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205115.230439:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205115.230443:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205115.230450:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513088:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205115.230453:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205115.230456:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230463:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205115.230464:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.230466:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205115.230468:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205115.230470:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205115.230473:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513216:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205115.230476:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205115.230477:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205115.230480:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205115.230481:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205115.230485:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb800. 02000000:00000001:1.0:1788205115.230487:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205115.230490:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205115.230493:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205115.230496:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205115.230499:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543513216, portal 4 00000100:00000001:1.0:1788205115.230501:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205115.230503:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205115.230508:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230515:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205115.230517:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205115.230519:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205115.230521:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543513216, offset 0 00000400:00000200:1.0:1788205115.230525:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205115.230529:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205115.230533:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205115.230536:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205115.230538:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205115.230540:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205115.230543:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205115.230546:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471358:471358:256:4294967295] 192.168.201.118@tcp LPNI seq info [471217:471217:8:4294967295] 00000400:00000200:1.0:1788205115.230550:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205115.230556:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205115.230560:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.230563:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205115.230567:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205115.230569:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205115.230573:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205115.230576:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205115.230580:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205115.230582:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205115.230585:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205115.230588:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205115.230591:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205115.230592:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205115.230595:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205115.230598:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230606:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.230609:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205140 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230615:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.230617:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205115.230620:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205115.230622:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205115.230626:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205115.230628:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321474101 00000100:00000001:1.0:1788205115.230630:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321475334 00000100:00000001:1.0:1788205115.230632:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321474101 00000100:00000001:1.0:1788205115.230634:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321475334 00000100:00000001:1.0:1788205115.230636:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205115.230638:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.230640:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205115.230644:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205115/real 1788205115] req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205115.230651:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205115.230653:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205115.230654:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205115.230657:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205115.230658:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205115.230661:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205115.230664:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205115.230672:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230678:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205115.230680:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230681:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.230683:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205115.230687:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230693:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205115.230696:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230702:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205115.230705:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230706:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205115.230708:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230709:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205115.230711:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205115.230713:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205115.230716:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205115.230719:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205115.230722:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205115.230731:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513216:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205115.230735:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230737:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.230741:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543512832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230748:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205115.230750:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205115.230752:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205115.230755:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205115.230756:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205115.230757:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205115.230760:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205115.230762:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205115.230765:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205115.230766:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205115.230768:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205115.230771:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205115.230772:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230774:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.230777:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543512960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230783:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205115.230784:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205115.230786:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205115.230800:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205115.230801:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205115.230802:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205115.230805:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205115.230806:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205115.230808:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205115.230810:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205115.230811:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205115.230813:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205115.230815:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230816:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.230820:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543513088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230826:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205115.230828:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205115.230830:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205115.230832:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205115.230834:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205115.230835:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205115.230837:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205115.230838:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205115.230840:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205115.230841:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205115.230843:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205115.230847:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205115.230848:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230849:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205115.230853:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543513216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205140 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205115.230860:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205115.230861:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205115.230862:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:1.0:1788205115.230865:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205115.230866:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205115.230868:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205115.230870:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205115.230871:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205115.230874:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205115.230875:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205115.230876:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205115.230878:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205115.230880:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230881:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205115.230886:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205115.230888:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205115.230890:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205115.230892:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205116.060227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205116.060231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205116.060235:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205116.060237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205116.060237:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205116.060245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205116.060246:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205116.060250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205116.060252:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205116.060254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205116.060254:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205116.060262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205116.060264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205116.060264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788205116.060266:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205116.060269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205116.060271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205116.060274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205116.060275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205116.060275:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205116.060280:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205116.060282:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205116.060287:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205116.060289:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205116.060295:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205116.060299:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205116.060343:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205116.060349:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205116.060353:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205116.572303:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321475334, at 4321475392 00000100:00000010:3.0:1788205116.572325:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647720. 00000100:00000040:2.0:1788205116.572857:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321475334, at 4321475392 00000100:00000010:2.0:1788205116.572867:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647600. 00000100:00000001:2.0:1788205117.084227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205117.084227:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205117.084233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205117.084237:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205117.084237:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205117.084239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205117.084241:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205117.084241:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205117.084248:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205117.084248:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205117.084251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205117.084254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205117.084254:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205117.084254:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205117.084259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205117.084259:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205117.084261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205117.084263:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205117.084263:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205117.084265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205117.084268:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205117.084268:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205117.084271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205117.084273:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205117.084273:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205117.084275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205117.084335:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205117.084339:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205117.084342:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205118.108217:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205118.108222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205118.108227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205118.108228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205118.108231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205118.108237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205118.108238:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205118.108241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205118.108244:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205118.108247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205118.108250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205118.108251:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205118.108255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205118.108256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205118.108259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205118.108261:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205118.108267:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205118.108361:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205118.108366:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205118.108369:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205118.108810:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205118.108815:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205118.108816:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205118.108819:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205118.108821:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205118.108823:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205118.108826:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205118.108829:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205118.108831:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205119.132196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205119.132196:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205119.132203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205119.132204:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205119.132208:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205119.132212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205119.132213:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205119.132215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205119.132218:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205119.132221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205119.132222:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205119.132224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205119.132227:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205119.132230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205119.132232:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205119.132233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205119.132239:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205119.132298:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205119.132302:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205119.132305:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205119.132532:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205119.132535:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205119.132536:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205119.132538:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205119.132540:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205119.132541:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205119.132542:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205119.132544:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205119.132546:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205120.156193:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205120.156193:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205120.156199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205120.156201:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205120.156205:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205120.156206:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205120.156210:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205120.156211:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:3.0:1788205120.156211:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205120.156212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205120.156216:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205120.156216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205120.156217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205120.156217:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205120.156221:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205120.156221:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205120.156222:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205120.156225:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205120.156225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205120.156225:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205120.156229:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205120.156229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205120.156229:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205120.156234:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205120.156234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.156234:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205120.156243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205120.156244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205120.156248:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205120.156250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205120.156254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205120.156255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205120.156313:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205120.156316:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205120.156319:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205120.348164:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205120.348172:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205120.348173:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205120.348176:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205120.348179:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205120.348184:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26806 00000400:00000200:3.0:1788205120.348192:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205120.348197:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205120.348199:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205120.348201:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348203:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348205:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348207:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205120.348209:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205120.348211:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205120.348213:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205120.348215:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205120.348217:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205120.348218:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205120.348220:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348221:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348223:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348224:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205120.348225:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205120.348227:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348229:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205120.348230:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205120.348233:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205120.348235:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348236:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205120.348238:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205120.348245:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205120.348247:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205120.348250:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205120.348251:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348253:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205120.348255:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205120.348258:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205120.348265:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412000. 00000100:00000001:3.0:1788205120.348272:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205120.348280:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205120.348290:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205120.348294:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205120.348299:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205120.348322:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205120.348325:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205120.348327:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205120.348328:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205120.348329:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205120.348331:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205120.348334:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26806 00000400:00000200:3.0:1788205120.348338:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205120.348341:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205120.348343:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205120.348345:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348348:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348350:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348352:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205120.348355:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205120.348356:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205120.348358:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205120.348360:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:1.0:1788205120.348362:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788205120.348363:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205120.348367:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205120.348368:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1788205120.348372:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205120.348373:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788205120.348377:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348382:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205120.348383:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205120.348384:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205120.348387:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205120.348392:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348396:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205120.348399:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205120.348403:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788205120.348405:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205120.348406:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205120.348408:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348411:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205120.348413:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1788205120.348414:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788205120.348417:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1788205120.348417:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348420:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.348423:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205120.348424:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000100:00100000:1.0:1788205120.348429:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543513344:192.168.201.118@tcp:250:kworker.0 00000020:00000040:3.0:1788205120.348434:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000001:1.0:1788205120.348435:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205120.348439:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205120.348440:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.348441:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205120.348442:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205120.348444:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205120.348445:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205120.348446:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.348449:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205120.348451:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411c00. 00000100:00000001:3.0:1788205120.348455:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205120.348457:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb800. 00000100:00100000:3.0:1788205120.348459:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205120.348461:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348468:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205120.348471:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205120.348472:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205120.348477:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00000040:3.0:1788205120.348482:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788205120.348486:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205120.348488:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205120.348492:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1788205120.348494:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543513344, portal 25 00000100:00080000:3.0:1788205120.348495:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205120.348497:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205120.348499:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788205120.348500:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205120.348503:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000001:3.0:1788205120.348504:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205120.348507:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26806 00000400:00000200:3.0:1788205120.348513:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:1.0:1788205120.348515:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205120.348517:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205120.348520:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205120.348524:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348528:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348531:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205120.348535:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788205120.348535:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205120.348537:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205120.348539:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205120.348541:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1788205120.348542:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000020:00000040:3.0:1788205120.348544:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000010:1.0:1788205120.348547:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000020:00000001:3.0:1788205120.348547:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205120.348549:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000200:1.0:1788205120.348553:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543513344, offset 0 00000100:00000040:3.0:1788205120.348553:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348557:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348559:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348560:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205120.348562:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:1.0:1788205120.348565:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205120.348566:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348570:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205120.348571:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205120.348573:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205120.348574:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348575:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205120.348578:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00100000:3.0:1788205120.348579:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205120.348583:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348584:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205120.348588:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000400:00000200:1.0:1788205120.348589:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000040:3.0:1788205120.348590:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205120.348593:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205120.348595:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.348596:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788205120.348597:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205120.348600:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205120.348601:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.348604:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000010:3.0:1788205120.348605:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412c00. 00000100:00000001:3.0:1788205120.348608:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.348610:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00100000:3.0:1788205120.348614:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205120.348618:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348623:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205120.348626:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.348628:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471359:471359:256:4294967295] 192.168.201.118@tcp LPNI seq info [471218:471218:8:4294967295] 00000100:00000040:3.0:1788205120.348631:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205120.348639:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205120.348642:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205120.348645:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205120.348647:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205120.348648:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205120.348650:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205120.348653:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205120.348657:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26806 00000400:00000200:1.0:1788205120.348659:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000400:00000200:3.0:1788205120.348670:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205120.348673:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:1.0:1788205120.348675:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:3.0:1788205120.348675:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205120.348677:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348681:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348682:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348683:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205120.348685:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000010:1.0:1788205120.348686:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000020:00000040:3.0:1788205120.348687:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205120.348689:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205120.348691:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205120.348693:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205120.348695:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:1.0:1788205120.348697:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205120.348713:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205120.348720:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205120.348728:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205120.348732:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.348735:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205120.348741:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205120.348745:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205120.348750:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205120.348752:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205120.348756:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205120.348761:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205120.348763:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348767:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348771:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348775:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205120.348777:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000001:1.0:1788205120.348779:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788205120.348779:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348782:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205120.348783:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205120.348785:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205120.348786:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348788:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205120.348789:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000040:1.0:1788205120.348792:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788205120.348792:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348793:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205120.348797:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788205120.348801:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205120.348805:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205120.348807:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348810:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205120.348813:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205120.348819:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.348823:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1788205120.348823:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.348831:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205120.348831:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788205120.348837:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.348840:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205120.348844:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321480453 00000100:00100000:3.0:1788205120.348847:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:1.0:1788205120.348851:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647ba0. 00000100:00000001:3.0:1788205120.348857:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205120.348859:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.348861:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000040:3.0:1788205120.348863:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.348869:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321480453 00000100:00000001:3.0:1788205120.348870:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205120.348873:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205120.348875:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205120.348876:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205120.348877:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205120.348879:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000010:1.0:1788205120.348882:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647e40. 00000100:00080000:3.0:1788205120.348882:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26806 00000400:00000200:3.0:1788205120.348886:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205120.348889:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000400:00000200:3.0:1788205120.348889:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205120.348893:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205120.348895:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1788205120.348896:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348900:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205120.348901:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205120.348902:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205120.348906:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788205120.348906:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205120.348910:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205120.348911:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205120.348912:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205120.348914:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00100000:1.0:1788205120.348916:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205120/real 1788205120] req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205120.348916:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205120.348920:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205120.348925:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348929:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205120.348932:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348936:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205120.348939:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000001:1.0:1788205120.348942:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788205120.348942:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205120.348944:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788205120.348946:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788205120.348947:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1788205120.348949:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 02000000:00000001:3.0:1788205120.348951:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205120.348954:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 02000000:00000001:3.0:1788205120.348954:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205120.348959:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205120.348959:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205120.348961:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205120.348963:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:3.0:1788205120.348966:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205120.348970:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205120.348971:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788205120.348977:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205120.348982:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000200:1.0:1788205120.348986:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205120.348988:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205120.348994:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205120.348996:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.348999:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788205120.349000:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349002:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205120.349004:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205120.349007:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788205120.349009:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205120.349010:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205120.349018:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:1.0:1788205120.349022:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205120.349024:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205120.349031:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1788205120.349038:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205120.349042:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205120.349046:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349052:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1788205120.349054:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349057:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349061:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.349064:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349069:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205120.349071:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349073:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1788205120.349077:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26816) 00000100:00080000:1.0:1788205120.349086:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205120.349115:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205120.349118:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205120.349122:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205120.349130:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543513344:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205120.349135:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349137:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.349139:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543513344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349146:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205120.349148:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205120.349151:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:1.0:1788205120.349154:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205120.349156:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205120.349157:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205120.349160:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205120.349162:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788205120.349165:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205120.349167:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205120.349169:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205120.349172:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205120.349173:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349175:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349181:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205120.349183:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349184:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205120.349186:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205120.349187:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205120.349190:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349197:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205120.349199:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.349201:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205120.349203:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205120.349205:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205120.349208:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513472:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205120.349211:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205120.349213:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205120.349214:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.349216:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205120.349219:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb800. 02000000:00000001:1.0:1788205120.349221:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.349224:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205120.349227:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205120.349230:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205120.349233:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543513472, portal 10 00000100:00000001:1.0:1788205120.349234:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205120.349236:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205120.349242:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349248:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205120.349251:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205120.349253:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205120.349255:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543513472, offset 0 00000400:00000200:1.0:1788205120.349259:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205120.349263:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205120.349267:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205120.349270:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205120.349272:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205120.349274:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205120.349277:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205120.349280:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471360:471360:256:4294967295] 192.168.201.118@tcp LPNI seq info [471219:471219:8:4294967295] 00000400:00000200:1.0:1788205120.349284:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205120.349290:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205120.349295:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.349298:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205120.349301:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205120.349304:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205120.349308:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205120.349311:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205120.349315:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.349316:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205120.349319:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205120.349323:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205120.349325:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205120.349327:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205120.349329:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205120.349333:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349340:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.349343:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349350:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349352:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205120.349354:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205120.349358:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205120.349361:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205120.349363:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321479187 00000100:00000001:1.0:1788205120.349365:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321480453 00000100:00000001:1.0:1788205120.349367:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321479187 00000100:00000001:1.0:1788205120.349370:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321480453 00000100:00000001:1.0:1788205120.349372:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205120.349374:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349376:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205120.349379:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205120/real 1788205120] req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205120.349387:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205120.349388:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205120.349390:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205120.349392:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205120.349394:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205120.349396:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205120.349399:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205120.349407:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349413:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205120.349414:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349416:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.349417:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205120.349420:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349426:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205120.349430:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349437:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205120.349440:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349441:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.349442:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349444:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205120.349446:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205120.349449:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205120.349452:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205120.349454:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205120.349458:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205120.349466:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513472:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205120.349470:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205120.349473:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349480:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205120.349481:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.349484:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205120.349485:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205120.349487:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205120.349490:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513600:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205120.349493:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205120.349495:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205120.349496:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.349497:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205120.349500:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205120.349502:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.349506:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205120.349508:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205120.349511:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205120.349514:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543513600, portal 10 00000100:00000001:1.0:1788205120.349516:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205120.349517:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205120.349522:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349528:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205120.349530:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205120.349532:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205120.349535:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543513600, offset 0 00000400:00000200:1.0:1788205120.349538:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205120.349542:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205120.349545:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205120.349548:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205120.349550:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205120.349552:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205120.349555:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205120.349558:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471361:471361:256:4294967295] 192.168.201.118@tcp LPNI seq info [471220:471220:8:4294967295] 00000400:00000200:1.0:1788205120.349562:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205120.349568:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205120.349572:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.349575:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205120.349578:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205120.349581:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205120.349585:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205120.349588:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205120.349591:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.349593:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205120.349596:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205120.349599:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205120.349602:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205120.349603:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205120.349606:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205120.349609:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349617:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.349620:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349626:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349628:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205120.349630:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205120.349633:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205120.349636:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205120.349638:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321479188 00000100:00000001:1.0:1788205120.349640:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321480453 00000100:00000001:1.0:1788205120.349642:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321479188 00000100:00000001:1.0:1788205120.349644:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321480453 00000100:00000001:1.0:1788205120.349646:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205120.349648:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349650:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205120.349653:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205120/real 1788205120] req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205120.349660:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205120.349662:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205120.349663:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205120.349665:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205120.349667:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205120.349669:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205120.349671:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205120.349679:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349685:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205120.349686:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349688:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.349689:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205120.349692:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349699:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205120.349722:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349728:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205120.349730:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349732:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.349733:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349735:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205120.349737:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205120.349740:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205120.349743:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205120.349745:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205120.349749:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205120.349756:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513600:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205120.349759:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205120.349762:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349769:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205120.349770:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.349773:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205120.349775:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205120.349777:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205120.349779:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513728:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205120.349783:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205120.349784:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205120.349786:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.349788:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205120.349791:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205120.349793:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.349796:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205120.349798:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205120.349801:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205120.349804:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543513728, portal 4 00000100:00000001:1.0:1788205120.349806:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205120.349807:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205120.349812:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349819:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205120.349821:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205120.349823:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205120.349825:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543513728, offset 0 00000400:00000200:1.0:1788205120.349828:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205120.349832:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205120.349835:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205120.349838:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205120.349840:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205120.349842:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205120.349844:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205120.349848:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471362:471362:256:4294967295] 192.168.201.118@tcp LPNI seq info [471221:471221:8:4294967295] 00000400:00000200:1.0:1788205120.349852:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205120.349858:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205120.349862:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.349865:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205120.349869:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205120.349872:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205120.349876:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205120.349878:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205120.349882:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.349884:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205120.349886:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205120.349889:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205120.349892:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205120.349894:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205120.349896:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205120.349900:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349907:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.349910:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349915:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349917:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205120.349920:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205120.349922:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205120.349925:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205120.349928:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321479197 00000100:00000001:1.0:1788205120.349930:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321480453 00000100:00000001:1.0:1788205120.349932:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321479197 00000100:00000001:1.0:1788205120.349934:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321480453 00000100:00000001:1.0:1788205120.349936:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205120.349938:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.349939:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205120.349942:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205120/real 1788205120] req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205120.349950:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205120.349952:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205120.349954:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205120.349956:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205120.349957:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205120.349959:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205120.349962:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205120.349970:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349976:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205120.349977:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.349979:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.349980:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205120.349983:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.349990:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205120.349993:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350000:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205120.350002:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350003:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.350005:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350006:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205120.350008:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205120.350010:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205120.350013:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205120.350015:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205120.350019:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205120.350027:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513728:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205120.350030:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205120.350033:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350039:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205120.350040:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.350042:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205120.350043:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205120.350045:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205120.350048:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513856:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205120.350051:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205120.350052:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205120.350054:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205120.350056:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205120.350058:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205120.350060:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205120.350062:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205120.350064:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205120.350067:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205120.350070:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543513856, portal 4 00000100:00000001:1.0:1788205120.350072:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205120.350073:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205120.350078:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350084:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205120.350087:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205120.350089:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205120.350107:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543513856, offset 0 00000400:00000200:1.0:1788205120.350111:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205120.350115:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205120.350118:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205120.350121:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205120.350123:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205120.350126:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205120.350128:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205120.350131:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471363:471363:256:4294967295] 192.168.201.118@tcp LPNI seq info [471222:471222:8:4294967295] 00000400:00000200:1.0:1788205120.350135:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205120.350141:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205120.350146:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.350148:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205120.350152:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205120.350154:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205120.350158:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205120.350161:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205120.350165:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205120.350167:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205120.350169:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205120.350173:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205120.350176:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205120.350177:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205120.350180:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205120.350183:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350191:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.350194:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205145 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350200:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.350201:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205120.350204:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205120.350207:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205120.350210:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205120.350212:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321479221 00000100:00000001:1.0:1788205120.350215:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321480453 00000100:00000001:1.0:1788205120.350217:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321479221 00000100:00000001:1.0:1788205120.350219:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321480453 00000100:00000001:1.0:1788205120.350221:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205120.350222:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.350224:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205120.350227:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205120/real 1788205120] req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205120.350234:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205120.350236:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205120.350237:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205120.350239:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205120.350241:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205120.350243:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205120.350246:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205120.350254:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350261:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205120.350262:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350263:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.350265:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205120.350268:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350275:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205120.350279:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350285:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205120.350288:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350289:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205120.350290:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350292:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205120.350294:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205120.350296:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205120.350299:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205120.350301:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205120.350304:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205120.350311:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543513856:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205120.350314:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350316:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.350319:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543513472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350325:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205120.350327:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205120.350329:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:1.0:1788205120.350331:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205120.350333:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205120.350334:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205120.350336:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205120.350338:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788205120.350341:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205120.350342:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205120.350343:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205120.350346:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205120.350347:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350349:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.350352:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543513600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350364:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205120.350365:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205120.350367:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205120.350369:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205120.350370:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205120.350371:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205120.350373:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205120.350374:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:1.0:1788205120.350375:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205120.350376:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205120.350377:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205120.350380:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205120.350381:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350382:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.350385:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543513728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350391:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205120.350392:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205120.350393:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205120.350395:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205120.350397:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205120.350398:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205120.350399:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205120.350401:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205120.350403:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205120.350404:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205120.350405:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205120.350407:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205120.350408:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350410:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205120.350413:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543513856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205145 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205120.350419:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205120.350420:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205120.350422:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205120.350423:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205120.350424:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205120.350426:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205120.350427:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205120.350428:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205120.350430:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205120.350431:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205120.350432:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205120.350435:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205120.350436:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350437:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205120.350441:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205120.350442:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205120.350444:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205120.350445:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205121.180236:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205121.180239:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205121.180240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205121.180244:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205121.180246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205121.180248:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205121.180248:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205121.180254:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205121.180257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205121.180259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205121.180260:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205121.180261:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205121.180266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205121.180270:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205121.180271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205121.180272:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205121.180283:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205121.180284:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205121.180291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205121.180297:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205121.180298:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205121.180298:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205121.180301:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205121.180303:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205121.180308:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205121.180309:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205121.180406:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205121.180412:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205121.180415:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205121.692248:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321480453, at 4321480512 00000100:00000040:2.0:1788205121.692249:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321480453, at 4321480512 00000100:00000010:3.0:1788205121.692275:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647e40. 00000100:00000010:2.0:1788205121.692284:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647ba0. 00000100:00000001:2.0:1788205122.204207:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205122.204215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205122.204216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205122.204220:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205122.204221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205122.204228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205122.204229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205122.204231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205122.204234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205122.204236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205122.204238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205122.204240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205122.204242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205122.204244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205122.204245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205122.204245:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205122.204247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205122.204252:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205122.204252:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205122.204256:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205122.204261:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205122.204264:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205122.204267:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205122.204271:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205122.204274:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205122.204276:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205122.204282:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205122.204288:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205122.204293:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205123.228205:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205123.228207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205123.228210:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205123.228213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205123.228214:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205123.228215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205123.228218:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205123.228219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205123.228221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205123.228225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205123.228226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205123.228227:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205123.228229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205123.228232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205123.228234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205123.228235:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205123.228237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205123.228239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205123.228240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205123.228240:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205123.228246:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205123.228246:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205123.228250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205123.228250:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205123.228255:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205123.228255:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205123.228310:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205123.228315:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205123.228319:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205124.252206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205124.252206:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205124.252215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205124.252216:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205124.252218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205124.252219:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205124.252227:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205124.252231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205124.252232:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205124.252236:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205124.252238:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205124.252239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205124.252245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205124.252245:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205124.252249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205124.252249:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205124.252256:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205124.252257:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252388:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205124.252391:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252396:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205124.252400:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205124.252402:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205124.252406:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252410:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205124.252412:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252415:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252417:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252421:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205124.252422:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205124.252423:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205124.252425:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252426:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205124.252428:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252430:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252431:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252435:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205124.252437:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205124.252438:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205124.252440:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252442:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205124.252443:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252444:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205124.252446:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205124.252503:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205124.252506:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205124.252508:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205125.276227:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205125.276231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205125.276238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205125.276239:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205125.276247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205125.276247:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205125.276250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205125.276255:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205125.276257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205125.276261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205125.276262:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205125.276266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205125.276267:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205125.276270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205125.276272:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205125.276277:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205125.276281:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205125.276436:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205125.276441:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205125.276445:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205125.276581:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205125.276586:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205125.276588:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205125.276591:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205125.276593:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205125.276596:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205125.277060:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205125.277064:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205125.277067:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205125.468237:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205125.468251:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205125.468254:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205125.468260:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205125.468265:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205125.468275:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26811 00000400:00000200:0.0:1788205125.468287:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205125.468295:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205125.468299:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205125.468303:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468305:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205125.468307:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468309:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205125.468313:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205125.468315:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205125.468317:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205125.468320:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205125.468322:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205125.468324:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205125.468327:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468328:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205125.468329:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468330:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205125.468333:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205125.468335:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468337:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205125.468339:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205125.468342:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205125.468345:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468346:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205125.468348:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205125.468358:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788205125.468361:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205125.468365:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205125.468367:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468371:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205125.468374:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205125.468379:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205125.468391:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788205125.468399:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205125.468410:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205125.468421:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205125.468426:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205125.468429:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205125.468453:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205125.468457:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205125.468461:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205125.468462:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205125.468464:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205125.468467:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205125.468471:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26811 00000400:00000200:0.0:1788205125.468476:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205125.468480:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205125.468483:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205125.468485:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468487:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205125.468488:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468490:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205125.468490:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205125.468495:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205125.468498:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788205125.468501:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205125.468503:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205125.468505:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1788205125.468505:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205125.468509:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205125.468513:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205125.468516:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205125.468521:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1788205125.468522:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468525:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205125.468525:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205125.468529:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468531:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:1.0:1788205125.468531:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1788205125.468534:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:1.0:1788205125.468535:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788205125.468539:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:1.0:1788205125.468540:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468545:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205125.468546:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543513984:192.168.201.118@tcp:250:kworker.0 02000000:00000001:0.0:1788205125.468548:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205125.468551:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205125.468555:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205125.468556:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788205125.468560:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.468560:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468563:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205125.468564:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1788205125.468568:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000010:1.0:1788205125.468571:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205125.468573:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788205125.468577:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468579:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205125.468582:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:0.0:1788205125.468583:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788205125.468586:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000400:00000010:1.0:1788205125.468588:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 02000000:00000001:0.0:1788205125.468589:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205125.468592:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468594:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205125.468595:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205125.468597:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205125.468600:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 02000000:00000010:0.0:1788205125.468601:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788205125.468606:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205125.468621:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205125.468627:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205125.468628:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205125.468631:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205125.468636:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543513984, portal 25 00000100:00000001:0.0:1788205125.468637:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.468642:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788205125.468643:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205125.468644:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114944 : -109419805436672 : ffff9c7bb75d9500) 00000100:00080000:0.0:1788205125.468649:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205125.468652:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205125.468653:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1788205125.468655:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205125.468656:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205125.468659:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26811 00000400:00000200:0.0:1788205125.468666:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205125.468672:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205125.468672:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205125.468677:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:0.0:1788205125.468678:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1788205125.468680:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000040:0.0:1788205125.468684:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000200:1.0:1788205125.468686:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543513984, offset 0 00000100:00000001:0.0:1788205125.468688:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205125.468692:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205125.468695:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468697:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205125.468701:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205125.468702:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788205125.468704:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:1.0:1788205125.468705:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:0.0:1788205125.468708:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205125.468713:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205125.468713:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:0.0:1788205125.468718:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:1.0:1788205125.468719:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1788205125.468725:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205125.468727:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:0.0:1788205125.468729:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205125.468732:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205125.468733:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788205125.468736:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205125.468738:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000040:0.0:1788205125.468740:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788205125.468744:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:1.0:1788205125.468745:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471364:471364:256:4294967295] 192.168.201.118@tcp LPNI seq info [471223:471223:8:4294967295] 00000100:00000001:0.0:1788205125.468749:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205125.468753:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:0.0:1788205125.468754:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205125.468758:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205125.468760:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468764:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205125.468767:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000008:00100000:0.0:1788205125.468768:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205125.468772:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468774:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205125.468778:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000010:0.0:1788205125.468780:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788205125.468783:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000800:00000010:1.0:1788205125.468783:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 02000000:00000001:0.0:1788205125.468785:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205125.468787:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468788:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205125.468789:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205125.468792:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205125.468792:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205125.468797:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 02000000:00000010:0.0:1788205125.468799:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00000001:0.0:1788205125.468804:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1788205125.468804:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205125.468809:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00100000:0.0:1788205125.468813:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788205125.468817:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.468821:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00000001:0.0:1788205125.468823:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205125.468826:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205125.468829:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205125.468829:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205125.468837:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1788205125.468841:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205125.468843:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205125.468844:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00080000:0.0:1788205125.468846:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:1.0:1788205125.468847:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:0.0:1788205125.468848:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205125.468851:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205125.468852:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788205125.468854:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205125.468858:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26811 00000100:00000200:1.0:1788205125.468859:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205125.468866:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205125.468868:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205125.468873:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205125.468877:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468879:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205125.468881:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205125.468881:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788205125.468885:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788205125.468886:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788205125.468888:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205125.468890:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205125.468892:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205125.468895:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788205125.468899:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205125.468899:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205125.468904:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205125.468904:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1788205125.468909:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205125.468910:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205125.468913:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205125.468913:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468918:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205125.468920:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:1.0:1788205125.468921:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205125.468925:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205125.468926:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321485573 00000100:00000001:0.0:1788205125.468927:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205125.468929:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:1.0:1788205125.468931:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16475a0. 02000000:00000001:0.0:1788205125.468932:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205125.468936:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468938:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205125.468940:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788205125.468941:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468942:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.468944:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000010:0.0:1788205125.468945:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788205125.468948:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205125.468950:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205125.468951:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.468952:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205125.468954:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205125.468954:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321485573 02000000:00000001:0.0:1788205125.468957:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205125.468962:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000010:1.0:1788205125.468962:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16478a0. 00000100:00000001:0.0:1788205125.468966:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.468969:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00100000:0.0:1788205125.468971:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205125.468975:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205125.468980:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.468985:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788205125.468988:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205125.468990:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205125.468992:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205125/real 1788205125] req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205125.468997:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205125.469025:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788205125.469025:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205125.469028:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1788205125.469031:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1788205125.469031:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205125.469035:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205125.469038:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00080000:0.0:1788205125.469039:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205125.469041:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1788205125.469042:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:1.0:1788205125.469046:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205125.469047:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205125.469055:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1788205125.469058:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205125.469062:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26811 00000100:00000001:1.0:1788205125.469065:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205125.469068:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469071:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:0.0:1788205125.469072:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205125.469073:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205125.469079:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205125.469082:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205125.469087:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205125.469111:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205125.469116:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469121:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205125.469123:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469126:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.469128:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205125.469132:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205125.469132:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205125.469135:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788205125.469138:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1788205125.469138:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:0.0:1788205125.469144:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00080000:1.0:1788205125.469144:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205125.469148:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788205125.469150:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788205125.469153:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788205125.469154:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205125.469157:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788205125.469159:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205125.469162:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205125.469167:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205125.469169:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00100000:1.0:1788205125.469169:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543513984:192.168.201.118@tcp:250:kworker.0 00000100:00000040:0.0:1788205125.469176:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205125.469176:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205125.469178:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205125.469179:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205125.469183:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205125.469184:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9500 x1875055543513984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205125.469189:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205125.469193:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000020:00000040:0.0:1788205125.469195:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 02000000:00000001:1.0:1788205125.469196:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205125.469200:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 00000100:00080000:0.0:1788205125.469202:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:1.0:1788205125.469205:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1788205125.469207:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205125.469210:0:256132:0:(genops.c:1239:class_import_put()) Process entered 02000000:00000001:0.0:1788205125.469211:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000020:00000040:1.0:1788205125.469214:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205125.469216:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788205125.469218:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000001:0.0:1788205125.469222:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205125.469223:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b800. 00000100:00000001:0.0:1788205125.469225:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205125.469228:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1788205125.469229:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000008:00000001:0.0:1788205125.469230:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.469233:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.469233:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205125.469237:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9500. 00000100:00000010:0.0:1788205125.469240:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000100:00000001:1.0:1788205125.469242:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000020:00000040:0.0:1788205125.469244:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788205125.469245:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788205125.469250:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205125.469251:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788205125.469254:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.469257:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205125.469259:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205125.469261:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.469261:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205125.469264:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.469266:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:0.0:1788205125.469267:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:1.0:1788205125.469267:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205125.469270:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205125.469274:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205125.469275:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205125.469283:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205125.469284:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205125.469286:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.469290:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205125.469292:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205125.469295:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788205125.469296:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205125.469299:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205125.469301:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514112:192.168.201.118@tcp:38:kworker.0 00000100:00000040:0.0:1788205125.469305:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469306:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205125.469309:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205125.469311:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.469315:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788205125.469318:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205125.469318:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205125.469322:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205125.469324:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26821) 00000400:00000200:1.0:1788205125.469327:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205125.469330:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205125.469334:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205125.469336:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543514112, portal 10 00000100:00000001:1.0:1788205125.469338:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205125.469339:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904113152 : -109419805438464 : ffff9c7bb75d8e00) 00000100:00000040:1.0:1788205125.469345:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469350:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205125.469352:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205125.469354:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205125.469356:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543514112, offset 0 00000400:00000200:1.0:1788205125.469360:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205125.469364:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205125.469368:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205125.469371:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205125.469373:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205125.469376:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205125.469378:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205125.469381:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471365:471365:256:4294967295] 192.168.201.118@tcp LPNI seq info [471224:471224:8:4294967295] 00000400:00000200:1.0:1788205125.469386:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205125.469392:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205125.469397:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.469399:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205125.469402:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205125.469404:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205125.469407:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205125.469408:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205125.469410:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.469412:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205125.469414:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205125.469416:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205125.469418:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205125.469419:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205125.469421:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205125.469423:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469429:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205125.469430:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469434:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.469435:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205125.469437:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205125.469441:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205125.469443:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205125.469444:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321484307 00000100:00000001:1.0:1788205125.469446:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321485573 00000100:00000001:1.0:1788205125.469448:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321484307 00000100:00000001:1.0:1788205125.469450:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321485573 00000100:00000001:1.0:1788205125.469451:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205125.469452:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.469453:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205125.469455:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205125/real 1788205125] req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205125.469460:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205125.469461:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205125.469462:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205125.469464:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205125.469465:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205125.469466:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205125.469468:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205125.469473:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469476:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205125.469477:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469478:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.469478:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205125.469480:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469483:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205125.469485:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469490:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205125.469491:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469492:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.469493:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469494:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205125.469495:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205125.469497:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205125.469498:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205125.469500:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205125.469502:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205125.469506:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514112:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205125.469509:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205125.469511:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469515:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205125.469515:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.469517:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205125.469518:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205125.469519:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205125.469520:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514240:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205125.469522:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205125.469523:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205125.469524:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.469526:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205125.469528:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205125.469529:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205125.469531:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205125.469532:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205125.469534:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205125.469536:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543514240, portal 10 00000100:00000001:1.0:1788205125.469537:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205125.469538:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904123904 : -109419805427712 : ffff9c7bb75db800) 00000100:00000040:1.0:1788205125.469542:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469546:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205125.469547:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205125.469548:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205125.469550:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543514240, offset 0 00000400:00000200:1.0:1788205125.469552:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205125.469554:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205125.469556:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205125.469557:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205125.469559:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205125.469561:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205125.469562:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205125.469564:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471366:471366:256:4294967295] 192.168.201.118@tcp LPNI seq info [471225:471225:8:4294967295] 00000400:00000200:1.0:1788205125.469567:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205125.469570:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205125.469573:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.469574:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205125.469577:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205125.469578:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205125.469581:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205125.469582:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205125.469586:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.469587:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205125.469590:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205125.469593:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205125.469596:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205125.469597:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205125.469599:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205125.469603:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469610:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205125.469631:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469638:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.469640:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205125.469644:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205125.469646:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205125.469648:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205125.469649:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321484308 00000100:00000001:1.0:1788205125.469651:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321485573 00000100:00000001:1.0:1788205125.469653:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321484308 00000100:00000001:1.0:1788205125.469654:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321485573 00000100:00000001:1.0:1788205125.469656:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205125.469657:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.469659:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205125.469661:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205125/real 1788205125] req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205125.469665:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205125.469667:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205125.469668:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205125.469669:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205125.469670:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205125.469671:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205125.469673:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205125.469678:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469681:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205125.469682:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469683:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.469684:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205125.469686:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469690:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205125.469692:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469696:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205125.469697:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469698:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.469699:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469700:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205125.469702:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205125.469704:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205125.469706:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205125.469708:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205125.469710:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205125.469715:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514240:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205125.469717:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205125.469718:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469722:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205125.469722:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.469724:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205125.469725:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205125.469726:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205125.469727:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514368:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205125.469729:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205125.469730:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205125.469731:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.469732:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205125.469735:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205125.469736:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205125.469738:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205125.469740:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205125.469742:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205125.469744:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543514368, portal 4 00000100:00000001:1.0:1788205125.469745:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205125.469746:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114048 : -109419805437568 : ffff9c7bb75d9180) 00000100:00000040:1.0:1788205125.469749:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469754:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205125.469755:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205125.469756:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205125.469758:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543514368, offset 0 00000400:00000200:1.0:1788205125.469761:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205125.469765:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205125.469767:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205125.469770:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205125.469772:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205125.469775:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205125.469777:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205125.469780:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471367:471367:256:4294967295] 192.168.201.118@tcp LPNI seq info [471226:471226:8:4294967295] 00000400:00000200:1.0:1788205125.469784:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205125.469789:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205125.469792:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.469795:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205125.469798:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205125.469801:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205125.469805:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205125.469807:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205125.469811:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.469813:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205125.469815:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205125.469819:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205125.469821:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205125.469823:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205125.469825:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205125.469828:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469834:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205125.469837:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469847:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.469848:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205125.469850:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205125.469853:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205125.469856:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205125.469858:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321484317 00000100:00000001:1.0:1788205125.469860:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321485573 00000100:00000001:1.0:1788205125.469862:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321484317 00000100:00000001:1.0:1788205125.469864:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321485573 00000100:00000001:1.0:1788205125.469865:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205125.469867:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.469868:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205125.469871:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205125/real 1788205125] req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205125.469876:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205125.469877:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205125.469878:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205125.469879:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205125.469880:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205125.469882:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205125.469883:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205125.469888:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469892:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205125.469893:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469894:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.469895:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205125.469896:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469901:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205125.469903:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469907:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205125.469908:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469909:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.469909:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469910:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205125.469911:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205125.469913:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205125.469914:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205125.469916:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205125.469918:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205125.469922:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514368:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205125.469924:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469925:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205125.469927:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8e00 x1875055543514112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469932:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205125.469933:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205125.469935:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205125.469937:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205125.469939:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205125.469940:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205125.469942:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205125.469944:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28800. 02000000:00000001:1.0:1788205125.469946:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205125.469948:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205125.469949:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8e00. 00000100:00000001:1.0:1788205125.469952:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205125.469953:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469954:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205125.469958:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db800 x1875055543514240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469965:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205125.469967:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205125.469969:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205125.469971:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205125.469972:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205125.469973:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205125.469976:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205125.469977:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29c00. 02000000:00000001:1.0:1788205125.469979:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205125.469980:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205125.469981:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db800. 00000100:00000001:1.0:1788205125.469983:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205125.469983:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.469985:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205125.469987:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9180 x1875055543514368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.469991:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205125.469992:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205125.469993:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205125.469994:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205125.469995:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205125.469996:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205125.469997:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205125.469997:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29400. 02000000:00000001:1.0:1788205125.469998:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205125.469999:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205125.469999:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9180. 00000100:00000001:1.0:1788205125.470000:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205125.470002:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.470003:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.470005:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205125.470006:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.470008:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205125.470009:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205125.470010:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205125.470012:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.470016:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205125.470017:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.470018:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205125.470019:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205125.470020:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205125.470022:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514496:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205125.470024:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205125.470025:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205125.470026:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205125.470027:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205125.470029:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205125.470030:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205125.470032:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205125.470034:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205125.470036:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205125.470038:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543514496, portal 4 00000100:00000001:1.0:1788205125.470040:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205125.470040:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904118528 : -109419805433088 : ffff9c7bb75da300) 00000100:00000040:1.0:1788205125.470044:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.470048:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205125.470049:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205125.470051:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205125.470052:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543514496, offset 0 00000400:00000200:1.0:1788205125.470054:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205125.470057:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205125.470059:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205125.470061:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205125.470063:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205125.470064:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205125.470066:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205125.470068:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471368:471368:256:4294967295] 192.168.201.118@tcp LPNI seq info [471227:471227:8:4294967295] 00000400:00000200:1.0:1788205125.470071:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205125.470074:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205125.470077:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.470079:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205125.470081:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205125.470084:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205125.470086:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205125.470088:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205125.470104:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205125.470105:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205125.470107:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205125.470109:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205125.470111:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205125.470112:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205125.470114:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205125.470116:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.470121:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205125.470123:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205150 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.470127:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.470128:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205125.470129:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205125.470131:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205125.470133:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205125.470135:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321484341 00000100:00000001:1.0:1788205125.470136:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321485573 00000100:00000001:1.0:1788205125.470138:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321484341 00000100:00000001:1.0:1788205125.470139:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321485573 00000100:00000001:1.0:1788205125.470140:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205125.470141:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.470142:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205125.470144:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205125/real 1788205125] req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205125.470148:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205125.470149:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205125.470150:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205125.470151:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205125.470152:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205125.470154:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205125.470156:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205125.470161:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.470164:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205125.470164:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.470165:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.470166:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205125.470168:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.470171:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205125.470173:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.470177:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205125.470178:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.470179:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205125.470180:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.470181:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205125.470183:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205125.470184:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205125.470186:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205125.470187:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205125.470189:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205125.470193:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514496:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205125.470195:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.470196:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205125.470197:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da300 x1875055543514496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205150 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205125.470201:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205125.470202:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205125.470204:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205125.470206:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205125.470207:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205125.470209:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205125.470211:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205125.470212:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:1.0:1788205125.470214:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205125.470216:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205125.470217:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da300. 00000100:00000001:1.0:1788205125.470219:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205125.470221:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.470222:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205125.470225:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205125.470227:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205125.470229:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205125.470231:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205126.300169:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205126.300169:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205126.300178:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205126.300179:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205126.300181:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205126.300181:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205126.300188:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205126.300188:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205126.300191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205126.300193:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205126.300193:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205126.300199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205126.300199:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205126.300199:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205126.300206:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205126.300206:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205126.300208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205126.300210:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205126.300210:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205126.300212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205126.300215:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205126.300215:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205126.300217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205126.300226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205126.300231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205126.300233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205126.300309:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205126.300315:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205126.300319:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205126.812233:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321485573, at 4321485632 00000100:00000040:3.0:1788205126.812233:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321485573, at 4321485632 00000100:00000010:2.0:1788205126.812251:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16475a0. 00000100:00000010:3.0:1788205126.812252:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16478a0. 00000100:00000001:3.0:1788205127.324234:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205127.324238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205127.324244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205127.324246:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205127.324251:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205127.324253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205127.324257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205127.324257:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205127.324263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205127.324263:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205127.324266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205127.324269:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205127.324272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205127.324275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205127.324275:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205127.324280:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205127.324285:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205127.324351:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205127.324356:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205127.324360:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205127.324412:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205127.324429:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205127.324431:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205127.324434:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205127.324436:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205127.324438:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205127.324441:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205127.324443:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205127.324445:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205128.348231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205128.348231:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205128.348241:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205128.348241:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205128.348243:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.348243:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205128.348249:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205128.348249:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205128.348253:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205128.348254:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205128.348258:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.348258:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205128.348261:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205128.348261:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205128.348265:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205128.348265:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205128.348268:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205128.348268:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205128.348391:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205128.348395:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205128.348399:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205128.348401:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205128.348404:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205128.348405:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205128.348409:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205128.348410:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205128.348470:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205128.348474:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205128.348476:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205129.372218:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205129.372218:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205129.372224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205129.372230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205129.372230:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205129.372231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205129.372235:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205129.372236:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205129.372239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205129.372242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205129.372243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205129.372243:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205129.372251:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205129.372251:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205129.372252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205129.372256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205129.372257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205129.372257:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205129.372261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205129.372263:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205129.372265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205129.372265:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205129.372269:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205129.372270:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205129.372274:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205129.372276:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205129.372337:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205129.372341:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205129.372345:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205130.396236:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205130.396243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205130.396248:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205130.396251:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205130.396252:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205130.396255:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205130.396261:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205130.396265:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205130.396266:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205130.396268:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205130.396271:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205130.396273:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205130.396276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205130.396278:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205130.396280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205130.396282:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205130.396287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205130.396287:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205130.396291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205130.396292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205130.396298:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205130.396299:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205130.396301:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205130.398610:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205130.398617:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205130.398623:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205130.399929:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205130.399935:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205130.399937:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205130.399943:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205130.399947:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205130.399950:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.399954:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.399956:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.399959:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205130.590615:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205130.590631:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205130.590634:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205130.590638:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205130.590643:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205130.590652:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26816 00000400:00000200:1.0:1788205130.590663:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205130.590671:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205130.590674:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205130.590677:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590681:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205130.590684:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590686:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205130.590691:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205130.590693:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205130.590696:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205130.590700:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205130.590702:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205130.590705:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205130.590708:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590710:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205130.590712:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590714:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205130.590717:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205130.590719:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590721:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205130.590724:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205130.590728:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205130.590730:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.590732:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205130.590735:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205130.590747:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788205130.590751:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205130.590755:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205130.590757:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.590761:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205130.590765:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205130.590770:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205130.590780:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9c00. 00000100:00000001:1.0:1788205130.590788:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205130.590797:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205130.590805:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205130.590810:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205130.590813:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205130.590833:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205130.590835:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205130.590838:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205130.590839:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205130.590841:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205130.590843:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205130.590845:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26816 00000400:00000200:1.0:1788205130.590849:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205130.590852:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205130.590855:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205130.590857:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590860:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205130.590862:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590864:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205130.590867:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205130.590868:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205130.590870:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205130.590873:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205130.590876:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205130.590878:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205130.590879:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1788205130.590882:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590886:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205130.590886:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788205130.590891:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205130.590891:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205130.590895:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205130.590898:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205130.590902:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000040:3.0:1788205130.590903:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205130.590905:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205130.590908:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205130.590916:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205130.590917:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788205130.590920:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.590922:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205130.590922:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788205130.590928:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205130.590929:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:1.0:1788205130.590931:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.590934:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205130.590935:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205130.590939:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 02000000:00000001:3.0:1788205130.590940:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205130.590942:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205130.590945:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205130.590947:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.590949:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788205130.590949:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543514624:192.168.201.118@tcp:250:kworker.0 02000000:00000001:1.0:1788205130.590951:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205130.590953:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205130.590958:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9800. 00000100:00000001:1.0:1788205130.590961:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205130.590965:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205130.590970:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205130.590971:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205130.590974:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205130.590979:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205130.590982:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205130.590984:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205130.590985:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205130.590986:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205130.590988:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205130.590990:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26816 00000400:00000200:1.0:1788205130.590992:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205130.590994:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205130.590995:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205130.590997:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.590998:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205130.590999:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591001:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205130.591002:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205130.591004:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205130.591005:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205130.591006:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205130.591008:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205130.591009:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205130.591011:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591012:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205130.591013:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591014:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205130.591015:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205130.591016:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591017:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205130.591018:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205130.591019:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205130.591020:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.591022:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205130.591023:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205130.591025:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.591025:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788205130.591027:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205130.591032:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 02000000:00000001:3.0:1788205130.591032:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1788205130.591036:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205130.591036:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205130.591039:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205130.591040:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205130.591041:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.591044:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205130.591044:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205130.591045:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205130.591048:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000010:1.0:1788205130.591049:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:3.0:1788205130.591053:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.591054:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205130.591058:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205130.591061:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205130.591064:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205130.591066:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205130.591069:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205130.591074:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205130.591076:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788205130.591078:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00080000:1.0:1788205130.591079:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205130.591080:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205130.591083:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205130.591086:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205130.591089:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26816 00000400:00000010:3.0:1788205130.591119:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205130.591130:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543514624, portal 25 00000400:00000200:1.0:1788205130.591134:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205130.591136:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205130.591139:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000400:00000200:1.0:1788205130.591140:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205130.591145:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205130.591149:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205130.591152:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591156:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205130.591160:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591165:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205130.591170:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205130.591172:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205130.591175:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788205130.591176:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1788205130.591178:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205130.591181:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205130.591182:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000020:00000001:1.0:1788205130.591184:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000010:3.0:1788205130.591188:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000040:1.0:1788205130.591190:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591193:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1788205130.591193:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543514624, offset 0 00000100:00000040:1.0:1788205130.591195:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591199:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205130.591201:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:3.0:1788205130.591205:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00080000:1.0:1788205130.591206:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591211:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205130.591214:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205130.591217:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205130.591217:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:1.0:1788205130.591222:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.591224:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205130.591225:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000008:00100000:1.0:1788205130.591228:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000400:00000200:3.0:1788205130.591231:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000001:1.0:1788205130.591232:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.591236:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.591237:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000010:1.0:1788205130.591242:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000400:00000200:3.0:1788205130.591242:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205130.591247:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000040:1.0:1788205130.591248:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205130.591252:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205130.591255:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.591255:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471369:471369:256:4294967295] 192.168.201.118@tcp LPNI seq info [471228:471228:8:4294967295] 00000100:00000001:1.0:1788205130.591259:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205130.591261:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205130.591264:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.591265:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000010:1.0:1788205130.591271:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb800. 00000100:00000001:1.0:1788205130.591276:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.591281:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00100000:1.0:1788205130.591285:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788205130.591293:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591299:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205130.591303:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205130.591304:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000100:00000040:1.0:1788205130.591309:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788205130.591318:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788205130.591324:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205130.591325:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00080000:1.0:1788205130.591329:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205130.591334:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000200:3.1:1788205130.591336:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:1.0:1788205130.591337:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205130.591339:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:3.1:1788205130.591343:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000100:00000001:1.0:1788205130.591344:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205130.591350:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26816 00000800:00000100:3.0:1788205130.591352:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.591358:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:1.0:1788205130.591359:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205130.591365:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205130.591366:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1788205130.591368:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205130.591373:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:1.0:1788205130.591377:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591380:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205130.591381:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000100:00000040:1.0:1788205130.591385:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000010:3.0:1788205130.591385:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:1.0:1788205130.591391:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205130.591392:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1788205130.591395:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205130.591398:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000200:3.0:1788205130.591399:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205130.591403:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205130.591407:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205130.591412:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205130.591415:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205130.591415:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205130.591422:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205130.591422:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205130.591427:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205130.591430:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205130.591435:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205130.591441:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205130.591447:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205130.591451:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.591452:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205130.591456:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205130.591456:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1788205130.591461:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205130.591463:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.591465:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.591466:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205130.591470:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205130.591474:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.591474:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205130.591477:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.591478:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321490696 00000100:00000010:1.0:1788205130.591481:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000100:00000010:3.0:1788205130.591484:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc240. 00000020:00000040:1.0:1788205130.591485:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205130.591489:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205130.591489:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205130.591491:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205130.591492:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205130.591493:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205130.591496:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bbc00. 00000100:00000001:1.0:1788205130.591500:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205130.591503:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205130.591508:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205130.591509:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205130.591511:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205130.591516:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205130.591519:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26827) 00000100:00000001:3.0:1788205130.591568:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205130.591574:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321490696 00000100:00000010:3.0:1788205130.591579:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc900. 00000100:00000001:3.0:1788205130.591583:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205130.591586:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205130.591590:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.591593:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205130.591599:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205130/real 1788205130] req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205130.591622:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205130.591624:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205130.591627:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205130.591629:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205130.591631:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205130.591634:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205130.591637:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205130.591647:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.591654:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205130.591656:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.591659:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.591661:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205130.591665:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.591673:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205130.591678:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.591685:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205130.591688:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.591690:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.591691:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.591694:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205130.591696:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205130.591699:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205130.591704:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205130.591706:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205130.591711:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205130.591719:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543514624:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205130.591724:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.591726:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.591729:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543514624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.591736:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205130.591739:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205130.591742:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205130.591745:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205130.591747:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205130.591749:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205130.591751:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205130.591753:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:3.0:1788205130.591758:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205130.591760:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205130.591761:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205130.591769:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205130.591770:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.591772:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.591782:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205130.591784:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.591786:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205130.591787:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205130.591789:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205130.591792:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.591800:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.591802:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.591804:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205130.591806:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205130.591808:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205130.591810:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514752:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205130.591813:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205130.591815:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205130.591817:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.591818:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205130.591821:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205130.591823:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.591827:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205130.591830:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205130.591834:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205130.591837:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543514752, portal 10 00000100:00000001:3.0:1788205130.591839:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205130.591841:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788205130.591848:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.591855:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205130.591857:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205130.591860:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205130.591862:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543514752, offset 0 00000400:00000200:3.0:1788205130.591866:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205130.591871:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205130.591875:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205130.591878:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205130.591880:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205130.591883:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205130.591885:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205130.591888:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471370:471370:256:4294967295] 192.168.201.118@tcp LPNI seq info [471229:471229:8:4294967295] 00000400:00000200:3.0:1788205130.591893:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205130.591899:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205130.591904:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.591907:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205130.591911:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205130.591913:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205130.591918:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205130.591921:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205130.591924:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.591926:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205130.591929:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205130.591932:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205130.591936:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205130.591938:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205130.591941:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205130.591944:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.591951:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.591953:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.591967:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.591968:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205130.591973:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205130.591977:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205130.591981:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205130.591983:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321489429 00000100:00000001:3.0:1788205130.591985:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321490696 00000100:00000001:3.0:1788205130.591988:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321489429 00000100:00000001:3.0:1788205130.591990:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321490696 00000100:00000001:3.0:1788205130.591992:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205130.591994:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.591996:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205130.592000:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205130/real 1788205130] req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205130.592008:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205130.592009:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205130.592011:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205130.592014:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205130.592015:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205130.592018:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205130.592021:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205130.592029:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592035:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205130.592037:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592038:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.592039:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205130.592043:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592050:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205130.592053:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592061:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205130.592063:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592065:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.592066:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592068:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205130.592070:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205130.592073:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205130.592075:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205130.592077:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205130.592079:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205130.592086:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514752:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205130.592110:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205130.592114:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592120:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.592121:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.592123:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205130.592124:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205130.592125:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205130.592127:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514880:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205130.592130:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205130.592131:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205130.592132:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.592134:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205130.592137:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205130.592138:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.592247:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205130.592250:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205130.592257:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205130.592261:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543514880, portal 10 00000100:00000001:3.0:1788205130.592263:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205130.592265:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205130.592271:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592277:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205130.592279:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205130.592281:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205130.592283:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543514880, offset 0 00000400:00000200:3.0:1788205130.592287:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205130.592291:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205130.592295:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205130.592298:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205130.592301:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205130.592303:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205130.592305:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205130.592308:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471371:471371:256:4294967295] 192.168.201.118@tcp LPNI seq info [471230:471230:8:4294967295] 00000400:00000200:3.0:1788205130.592312:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205130.592318:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205130.592323:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.592326:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205130.592330:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205130.592333:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205130.592336:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205130.592338:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205130.592341:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.592342:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205130.592344:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205130.592346:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205130.592348:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205130.592351:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205130.592352:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205130.592355:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592360:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.592362:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592366:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.592367:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205130.592369:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205130.592371:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205130.592373:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205130.592375:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321489431 00000100:00000001:3.0:1788205130.592377:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321490696 00000100:00000001:3.0:1788205130.592378:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321489431 00000100:00000001:3.0:1788205130.592380:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321490696 00000100:00000001:3.0:1788205130.592381:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205130.592382:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.592383:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205130.592385:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205130/real 1788205130] req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205130.592390:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205130.592391:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205130.592392:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205130.592394:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205130.592395:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205130.592396:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205130.592398:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205130.592403:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592410:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205130.592411:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592413:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.592414:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205130.592417:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592423:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205130.592427:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592433:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205130.592435:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592436:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.592438:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592440:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205130.592442:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205130.592445:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205130.592448:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205130.592451:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205130.592455:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205130.592461:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543514880:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205130.592463:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205130.592465:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592470:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.592471:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.592473:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205130.592474:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205130.592475:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205130.592477:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515008:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205130.592479:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205130.592481:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205130.592483:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.592485:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205130.592598:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205130.592602:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.592607:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205130.592610:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205130.592615:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205130.592618:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543515008, portal 4 00000100:00000001:3.0:1788205130.592621:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205130.592622:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205130.592629:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592635:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205130.592638:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205130.592640:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205130.592642:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543515008, offset 0 00000400:00000200:3.0:1788205130.592645:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205130.592650:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205130.592654:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205130.592656:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205130.592669:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205130.592672:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205130.592674:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205130.592677:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471372:471372:256:4294967295] 192.168.201.118@tcp LPNI seq info [471231:471231:8:4294967295] 00000400:00000200:3.0:1788205130.592682:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205130.592688:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205130.592694:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.592697:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205130.592701:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205130.592704:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205130.592709:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205130.592711:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205130.592715:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.592717:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205130.592720:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205130.592722:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205130.592725:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205130.592726:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205130.592728:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205130.592735:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592741:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.592743:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592747:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.592748:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205130.592750:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205130.592752:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205130.592755:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205130.592756:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321489440 00000100:00000001:3.0:1788205130.592758:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321490696 00000100:00000001:3.0:1788205130.592760:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321489440 00000100:00000001:3.0:1788205130.592761:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321490696 00000100:00000001:3.0:1788205130.592762:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205130.592763:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.592764:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205130.592766:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205130/real 1788205130] req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205130.592772:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205130.592773:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205130.592774:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205130.592775:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205130.592776:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205130.592777:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205130.592779:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205130.592784:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592787:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205130.592788:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592789:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.592790:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205130.592792:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592795:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205130.592797:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592801:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205130.592803:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592804:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.592805:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.592806:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205130.592807:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205130.592809:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205130.592811:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205130.592812:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205130.592815:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205130.592819:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515008:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205130.592821:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205130.592823:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592827:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205130.592827:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.592829:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205130.592830:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205130.592831:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205130.592832:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515136:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205130.592834:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205130.592835:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205130.592836:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205130.592837:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205130.592839:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205130.592840:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205130.592842:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205130.592844:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205130.592853:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205130.592856:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543515136, portal 4 00000100:00000001:3.0:1788205130.592858:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205130.592859:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205130.592865:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592871:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205130.592873:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205130.592875:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205130.592878:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543515136, offset 0 00000400:00000200:3.0:1788205130.592882:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205130.592886:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205130.592890:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205130.592892:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205130.592895:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205130.592897:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205130.592899:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205130.592902:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471373:471373:256:4294967295] 192.168.201.118@tcp LPNI seq info [471232:471232:8:4294967295] 00000400:00000200:3.0:1788205130.592905:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205130.592910:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205130.592913:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.592915:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205130.592918:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205130.592920:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205130.592922:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205130.592924:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205130.592929:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205130.592931:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205130.592938:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205130.592941:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205130.592945:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205130.592946:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205130.592948:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205130.592952:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592974:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.592977:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205155 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.592983:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.592984:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205130.592987:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205130.592990:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205130.592993:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205130.592995:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321489463 00000100:00000001:3.0:1788205130.592998:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321490696 00000100:00000001:3.0:1788205130.593000:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321489463 00000100:00000001:3.0:1788205130.593002:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321490696 00000100:00000001:3.0:1788205130.593004:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205130.593006:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.593008:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205130.593016:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205130/real 1788205130] req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205130.593023:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205130.593025:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205130.593027:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205130.593029:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205130.593030:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205130.593032:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205130.593036:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205130.593044:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.593050:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205130.593051:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593053:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.593055:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205130.593058:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.593065:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205130.593069:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.593075:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205130.593078:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593079:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205130.593081:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593082:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205130.593084:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205130.593087:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205130.593109:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205130.593112:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205130.593116:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205130.593124:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515136:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205130.593127:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593129:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.593132:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543514752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.593138:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205130.593140:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205130.593143:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205130.593145:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205130.593147:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205130.593148:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205130.593158:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205130.593161:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:3.0:1788205130.593163:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205130.593165:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205130.593166:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788205130.593171:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205130.593173:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593175:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.593178:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543514880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.593191:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205130.593192:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205130.593193:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205130.593196:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205130.593197:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205130.593198:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205130.593200:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205130.593201:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:3.0:1788205130.593203:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205130.593204:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205130.593206:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205130.593208:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205130.593209:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593211:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.593213:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543515008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.593220:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205130.593221:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205130.593223:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205130.593225:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205130.593226:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205130.593227:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205130.593229:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205130.593230:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788205130.593232:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205130.593233:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205130.593235:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205130.593242:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205130.593243:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593244:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205130.593247:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543515136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205155 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205130.593255:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205130.593256:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205130.593258:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205130.593260:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205130.593261:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205130.593262:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205130.593265:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205130.593266:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:3.0:1788205130.593268:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205130.593269:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205130.593270:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205130.593272:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205130.593273:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593275:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205130.593280:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205130.593282:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205130.593284:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205130.593286:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205131.420203:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205131.420203:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205131.420208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205131.420213:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205131.420213:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205131.420214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205131.420217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205131.420218:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205131.420224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205131.420226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205131.420226:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205131.420227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205131.420234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205131.420234:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205131.420235:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205131.420237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205131.420240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205131.420240:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205131.420243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205131.420246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205131.420247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205131.420247:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205131.420251:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205131.420252:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205131.420256:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205131.420257:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205131.420305:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205131.420308:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205131.420311:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205131.932249:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321490696, at 4321490752 00000100:00000040:3.0:1788205131.932249:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321490696, at 4321490752 00000100:00000010:2.0:1788205131.932267:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc240. 00000100:00000010:3.0:1788205131.932271:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc900. 00000100:00000001:2.0:1788205132.444235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205132.444240:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205132.444251:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205132.444251:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205132.444254:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205132.444255:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205132.444262:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205132.444262:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205132.444269:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205132.444269:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205132.444275:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205132.444275:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205132.444281:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205132.444282:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205132.444287:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205132.444288:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205132.444292:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205132.444293:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205132.445580:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205132.445585:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205132.445589:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205132.445590:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205132.445592:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205132.445593:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205132.445595:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205132.445596:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205132.445688:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205132.445692:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205132.445696:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205133.468191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205133.468199:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205133.468199:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205133.468201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205133.468207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205133.468207:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205133.468210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205133.468211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205133.468211:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205133.468214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205133.468218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205133.468218:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205133.468219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205133.468221:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205133.468225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205133.468225:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205133.468227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205133.468230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205133.468230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205133.468230:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205133.468236:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205133.468236:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205133.468240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205133.468240:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205133.468244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205133.468244:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205133.468280:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205133.468285:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205133.468289:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205134.492237:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205134.492243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205134.492247:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205134.492248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.492251:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205134.492258:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205134.492259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205134.492265:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205134.492265:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205134.492269:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205134.492270:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205134.492273:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.492276:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205134.492279:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788205134.492281:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205134.492282:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492287:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205134.492287:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492293:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205134.492301:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205134.492303:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205134.492304:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492305:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205134.492306:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492308:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492311:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492316:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205134.492318:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205134.492319:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205134.492321:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492323:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205134.492324:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492326:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205134.492328:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205134.492543:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000100:00000001:2.0:1788205134.492548:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000020:00000001:1.0:1788205134.492549:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205134.492554:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205134.492555:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205134.492559:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205134.492563:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205134.492566:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205134.492569:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205134.492572:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205134.492574:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205134.492576:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205135.516218:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205135.516218:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205135.516224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205135.516229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205135.516229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205135.516229:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205135.516233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.516233:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205135.516238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205135.516239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205135.516240:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205135.516241:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205135.516244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205135.516246:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205135.516247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205135.516247:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205135.516249:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205135.516251:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205135.516253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205135.516253:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205135.516256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205135.516256:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205135.516257:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205135.516261:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.516262:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205135.516266:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205135.516342:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205135.516346:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205135.516349:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205135.708312:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205135.708325:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205135.708329:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205135.708334:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205135.708340:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205135.708349:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26822 00000400:00000200:0.0:1788205135.708362:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205135.708371:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205135.708375:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205135.708378:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708382:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205135.708385:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708389:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205135.708394:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205135.708396:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205135.708400:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205135.708404:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205135.708407:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205135.708410:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205135.708413:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708415:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205135.708417:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708420:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205135.708423:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205135.708427:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708430:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205135.708449:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205135.708454:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205135.708457:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.708460:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205135.708463:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205135.708476:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788205135.708480:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205135.708485:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205135.708486:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.708490:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205135.708494:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205135.708500:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205135.708510:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788205135.708518:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205135.708528:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205135.708603:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205135.708609:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205135.708616:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205135.708647:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205135.708651:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205135.708655:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205135.708656:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205135.708658:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205135.708661:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205135.708664:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26822 00000400:00000200:0.0:1788205135.708669:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205135.708672:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205135.708675:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205135.708678:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708680:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205135.708682:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708684:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205135.708687:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205135.708688:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205135.708690:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205135.708692:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205135.708694:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205135.708697:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788205135.708699:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708701:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205135.708703:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708705:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205135.708707:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788205135.708710:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708711:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205135.708714:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205135.708722:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205135.708724:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.708726:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205135.708732:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205135.708735:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.708737:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.708741:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:0.0:1788205135.708742:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788205135.708746:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000001:3.0:1788205135.708748:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788205135.708750:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205135.708753:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.708753:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205135.708757:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205135.708759:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205135.708762:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205135.708764:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788205135.708767:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788205135.708773:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.708778:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205135.708782:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205135.708784:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788205135.708787:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205135.708791:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205135.708799:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205135.708801:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205135.708805:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205135.708813:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205135.708817:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205135.708820:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205135.708822:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205135.708823:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205135.708826:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205135.708829:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26822 00000400:00000200:0.0:1788205135.708833:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205135.708836:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205135.708839:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205135.708843:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708845:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205135.708847:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708850:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:3.0:1788205135.708851:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205135.708854:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205135.708857:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788205135.708860:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:3.0:1788205135.708860:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543515264:192.168.201.118@tcp:250:kworker.0 00000020:00000040:0.0:1788205135.708862:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205135.708867:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:3.0:1788205135.708869:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1788205135.708871:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788205135.708876:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:3.0:1788205135.708877:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788205135.708878:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205135.708881:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:3.0:1788205135.708882:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.708883:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205135.708887:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 02000000:00000001:3.0:1788205135.708887:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788205135.708891:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205135.708894:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205135.708895:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:0.0:1788205135.708896:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205135.708900:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205135.708902:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205135.708905:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.708908:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205135.708908:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00100000:0.0:1788205135.708911:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205135.708914:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205135.708915:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:0.0:1788205135.708917:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205135.708923:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788205135.708925:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000010:3.0:1788205135.708928:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 02000000:00000001:0.0:1788205135.708929:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205135.708931:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.708934:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205135.708935:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205135.708937:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205135.708940:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543515264, portal 25 02000000:00000010:0.0:1788205135.708942:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:3.0:1788205135.708945:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788205135.708948:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.708949:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904111360 : -109419805440256 : ffff9c7bb75d8700) 00000100:00100000:0.0:1788205135.708955:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788205135.708960:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205135.708973:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205135.708977:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.708977:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205135.708984:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205135.708984:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205135.708990:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205135.708996:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543515264, offset 0 00000100:00000001:0.0:1788205135.708997:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205135.709003:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205135.709008:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788205135.709009:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709011:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205135.709014:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205135.709017:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205135.709021:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26822 00000400:00000200:3.0:1788205135.709024:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205135.709029:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205135.709033:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205135.709036:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205135.709037:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000040:0.0:1788205135.709042:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709045:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205135.709046:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1788205135.709050:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709053:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205135.709054:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000020:00000001:0.0:1788205135.709057:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205135.709060:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000400:00000200:3.0:1788205135.709062:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000001:0.0:1788205135.709063:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205135.709067:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000400:00000200:3.0:1788205135.709068:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000001:0.0:1788205135.709072:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788205135.709076:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788205135.709077:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471374:471374:256:4294967295] 192.168.201.118@tcp LPNI seq info [471233:471233:8:4294967295] 00000100:00000040:0.0:1788205135.709082:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709087:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205135.709087:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:0.0:1788205135.709119:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709122:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205135.709125:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788205135.709128:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709131:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205135.709133:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205135.709136:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205135.709139:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.709143:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205135.709145:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788205135.709148:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.709152:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205135.709157:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000010:0.0:1788205135.709158:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788205135.709162:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205135.709164:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205135.709165:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.709168:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205135.709169:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205135.709171:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205135.709175:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788205135.709178:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205135.709182:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205135.709189:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205135.709191:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205135.709195:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205135.709202:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205135.709205:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205135.709209:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205135.709210:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205135.709212:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205135.709216:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205135.709218:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26822 00000400:00000200:0.0:1788205135.709222:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205135.709225:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205135.709227:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205135.709230:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709232:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205135.709234:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709236:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205135.709238:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205135.709240:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788205135.709241:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205135.709244:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205135.709246:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205135.709248:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788205135.709251:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709253:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205135.709255:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709257:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205135.709258:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788205135.709261:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205135.709264:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205135.709265:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205135.709268:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205135.709271:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.709273:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205135.709275:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205135.709277:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.709279:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205135.709283:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788205135.709286:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205135.709288:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205135.709289:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205135.709291:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205135.709292:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205135.709294:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205135.709298:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:0.0:1788205135.709301:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205135.709306:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205135.709312:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205135.709315:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205135.709318:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205135.709326:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205135.709330:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26832) 00000800:00000200:3.0:1788205135.709773:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.709780:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205135.709788:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205135.709793:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205135.709799:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205135.710017:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205135.710023:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.710027:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205135.710035:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205135.710040:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205135.710048:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205135.710051:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205135.710056:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205135.710061:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.710072:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.710076:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.710084:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.710086:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205135.710122:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205135.710128:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205135.710132:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321495815 00000100:00000010:3.0:1788205135.710138:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc900. 00000100:00000001:3.0:1788205135.710145:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205135.710151:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321495815 00000100:00000010:3.0:1788205135.710154:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc540. 00000100:00000001:3.0:1788205135.710157:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205135.710160:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205135.711542:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.711550:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205135.711561:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205135/real 1788205135] req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205135.711576:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205135.711579:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205135.711583:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205135.711587:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205135.711589:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205135.711593:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205135.711596:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205135.711604:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.711611:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205135.711613:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.711616:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.711618:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205135.711622:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.711629:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205135.711633:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.711641:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205135.711645:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.711646:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.711647:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.711650:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205135.711652:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205135.711656:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205135.711661:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205135.711663:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205135.711667:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205135.711674:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543515264:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205135.711682:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.711684:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.711687:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8700 x1875055543515264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.711694:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205135.711698:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205135.711701:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205135.711705:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205135.711708:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205135.711710:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205135.711713:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205135.711716:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a800. 02000000:00000001:3.0:1788205135.711721:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205135.711723:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205135.711725:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8700. 00000100:00000001:3.0:1788205135.711728:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205135.711729:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.711732:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.711742:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205135.711744:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.711747:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205135.711749:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205135.711752:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205135.711757:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.711765:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205135.711767:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205135.711771:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205135.711775:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205135.711778:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205135.711781:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515392:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205135.711785:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205135.711791:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205135.711794:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205135.711797:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205135.711804:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205135.711806:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205135.711812:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205135.711816:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205135.711821:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205135.711826:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543515392, portal 10 00000100:00000001:3.0:1788205135.711829:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205135.711830:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904110464 : -109419805441152 : ffff9c7bb75d8380) 00000100:00000040:3.0:1788205135.711838:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.711844:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205135.711847:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205135.711850:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205135.711852:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543515392, offset 0 00000400:00000200:3.0:1788205135.711859:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205135.711866:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205135.711875:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205135.711881:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205135.711886:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205135.711890:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205135.711893:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205135.711898:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471375:471375:256:4294967295] 192.168.201.118@tcp LPNI seq info [471234:471234:8:4294967295] 00000400:00000200:3.0:1788205135.711902:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205135.711912:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205135.711917:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.711923:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205135.711930:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205135.711934:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205135.711939:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205135.711942:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205135.711946:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.711948:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205135.711952:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205135.711956:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205135.711961:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205135.711962:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205135.711965:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205135.711969:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.711976:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.711979:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.711985:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.711986:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205135.711992:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205135.711996:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205135.711999:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205135.712003:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321494549 00000100:00000001:3.0:1788205135.712007:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321495815 00000100:00000001:3.0:1788205135.712009:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321494549 00000100:00000001:3.0:1788205135.712011:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321495815 00000100:00000001:3.0:1788205135.712013:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205135.712015:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.712016:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205135.712020:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205135/real 1788205135] req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205135.712027:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205135.712028:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205135.712030:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205135.712032:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205135.712034:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205135.712036:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205135.712039:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205135.712047:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712053:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205135.712055:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.712056:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.712058:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205135.712061:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712069:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205135.712073:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712080:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205135.712082:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.712084:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.712085:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.712086:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205135.712089:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205135.712110:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205135.712113:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205135.712116:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205135.712120:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205135.712127:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515392:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205135.712131:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205135.712135:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712146:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205135.712148:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205135.712150:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205135.712151:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205135.712153:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205135.712156:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515520:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205135.712160:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205135.712550:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205135.712553:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205135.712555:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205135.712560:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205135.712563:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205135.712568:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205135.712571:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205135.712576:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205135.712580:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543515520, portal 10 00000100:00000001:3.0:1788205135.712583:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205135.712584:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904109568 : -109419805442048 : ffff9c7bb75d8000) 00000100:00000040:3.0:1788205135.712592:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712598:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205135.712601:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205135.712603:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205135.712606:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543515520, offset 0 00000400:00000200:3.0:1788205135.712611:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205135.712615:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205135.712620:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205135.712624:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205135.712627:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205135.712630:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205135.712632:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205135.712636:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471376:471376:256:4294967295] 192.168.201.118@tcp LPNI seq info [471235:471235:8:4294967295] 00000400:00000200:3.0:1788205135.712640:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205135.712647:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205135.712652:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.712656:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205135.712661:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205135.712664:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205135.712669:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205135.712672:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205135.712676:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.712678:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205135.712682:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205135.712685:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205135.712689:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205135.712691:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205135.712694:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205135.712698:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712705:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.712905:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712912:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.712913:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205135.712919:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205135.712923:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205135.712926:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205135.712928:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321494551 00000100:00000001:3.0:1788205135.712931:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321495815 00000100:00000001:3.0:1788205135.712933:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321494551 00000100:00000001:3.0:1788205135.712935:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321495815 00000100:00000001:3.0:1788205135.712937:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205135.712940:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.712942:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205135.712946:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205135/real 1788205135] req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205135.712954:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205135.712955:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205135.712957:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205135.712960:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205135.712961:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205135.712964:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205135.712967:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205135.712974:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712980:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205135.712981:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.712983:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.712984:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205135.712987:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.712993:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205135.712997:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713003:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205135.713005:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713007:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.713008:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713010:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205135.713013:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205135.713015:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205135.713020:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205135.713021:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205135.713025:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205135.713032:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515520:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205135.713035:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205135.713040:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713046:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205135.713047:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205135.713050:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205135.713052:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205135.713053:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205135.713056:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515648:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205135.713059:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205135.713062:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205135.713064:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205135.713066:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205135.713070:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205135.713072:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205135.713075:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205135.713077:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205135.713081:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205135.713084:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543515648, portal 4 00000100:00000001:3.0:1788205135.713086:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205135.713087:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904122112 : -109419805429504 : ffff9c7bb75db100) 00000100:00000040:3.0:1788205135.713245:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713252:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205135.713254:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205135.713257:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205135.713260:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543515648, offset 0 00000400:00000200:3.0:1788205135.713265:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205135.713269:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205135.713272:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205135.713275:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205135.713278:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205135.713281:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205135.713283:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205135.713286:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471377:471377:256:4294967295] 192.168.201.118@tcp LPNI seq info [471236:471236:8:4294967295] 00000400:00000200:3.0:1788205135.713290:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205135.713296:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205135.713300:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.713304:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205135.713308:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205135.713311:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205135.713315:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205135.713317:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205135.713321:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.713323:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205135.713326:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205135.713329:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205135.713333:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205135.713334:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205135.713337:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205135.713340:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713348:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.713351:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713357:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.713358:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205135.713362:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205135.713365:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205135.713368:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205135.713370:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321494561 00000100:00000001:3.0:1788205135.713372:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321495815 00000100:00000001:3.0:1788205135.713374:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321494561 00000100:00000001:3.0:1788205135.713376:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321495815 00000100:00000001:3.0:1788205135.713378:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205135.713380:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.713382:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205135.713385:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205135/real 1788205135] req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205135.713392:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205135.713393:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205135.713395:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205135.713398:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205135.713399:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205135.713401:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205135.713403:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205135.713410:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713415:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205135.713416:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713418:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.713419:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205135.713422:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713428:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205135.713432:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713447:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205135.713449:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713451:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.713452:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713454:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205135.713456:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205135.713459:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205135.713462:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205135.713464:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205135.713468:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205135.713475:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515648:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205135.713479:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205135.713483:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713488:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205135.713490:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205135.713492:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205135.713494:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205135.713496:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205135.713498:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515776:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205135.713501:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205135.713504:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205135.713505:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205135.713507:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205135.713511:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205135.713513:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205135.713516:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205135.713519:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205135.713522:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205135.713525:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543515776, portal 4 00000100:00000001:3.0:1788205135.713527:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205135.713528:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904121216 : -109419805430400 : ffff9c7bb75dad80) 00000100:00000040:3.0:1788205135.713535:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713540:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205135.713543:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205135.713545:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205135.713547:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543515776, offset 0 00000400:00000200:3.0:1788205135.713551:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205135.713555:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205135.713559:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205135.713562:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205135.713565:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205135.713567:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205135.713570:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205135.713573:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471378:471378:256:4294967295] 192.168.201.118@tcp LPNI seq info [471237:471237:8:4294967295] 00000400:00000200:3.0:1788205135.713577:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205135.713583:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205135.713587:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.713591:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205135.713595:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205135.713597:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205135.713601:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205135.713604:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205135.713608:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205135.713610:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205135.713613:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205135.713617:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205135.713620:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205135.713622:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205135.713624:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205135.713628:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713635:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.713638:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205160 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713643:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.713645:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205135.713648:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205135.713651:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205135.713655:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205135.713657:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321494584 00000100:00000001:3.0:1788205135.713659:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321495815 00000100:00000001:3.0:1788205135.713662:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321494584 00000100:00000001:3.0:1788205135.713664:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321495815 00000100:00000001:3.0:1788205135.713666:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205135.713668:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.713670:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205135.713673:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205135/real 1788205135] req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205135.713680:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205135.713682:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205135.713683:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205135.713685:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205135.713687:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205135.713688:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205135.713691:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205135.713698:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713768:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205135.713770:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713772:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.713774:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205135.713778:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713785:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205135.713789:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713796:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205135.713799:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713860:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205135.713862:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713864:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205135.713866:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205135.713869:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205135.713873:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205135.713875:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205135.713880:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205135.713886:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543515776:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205135.713889:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713890:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.713892:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8380 x1875055543515392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713897:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205135.713899:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205135.713902:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205135.713904:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205135.713905:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205135.713907:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205135.713909:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205135.713911:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2ac00. 02000000:00000001:3.0:1788205135.713914:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205135.713915:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205135.713916:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8380. 00000100:00000001:3.0:1788205135.713919:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205135.713920:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713921:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.713923:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8000 x1875055543515520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713927:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205135.713928:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205135.713929:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205135.713930:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205135.713931:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205135.713932:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205135.713934:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205135.713934:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28c00. 02000000:00000001:3.0:1788205135.713936:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205135.713936:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205135.713937:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8000. 00000100:00000001:3.0:1788205135.713938:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205135.713939:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713939:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.713941:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db100 x1875055543515648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.713944:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205135.713945:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205135.713945:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205135.713946:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205135.713947:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205135.713948:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205135.713949:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205135.713950:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:3.0:1788205135.713951:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205135.713952:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205135.713953:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db100. 00000100:00000001:3.0:1788205135.713954:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205135.713955:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.713956:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205135.714074:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75dad80 x1875055543515776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205160 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205135.714081:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205135.714083:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205135.714084:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205135.714087:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205135.714088:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205135.714149:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205135.714151:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205135.714152:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b400. 02000000:00000001:3.0:1788205135.714154:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205135.714155:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205135.714156:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dad80. 00000100:00000001:3.0:1788205135.714158:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205135.714158:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.714160:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205135.714217:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205135.714219:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205135.714220:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205135.714222:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205136.540260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205136.540270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205136.540277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205136.540278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205136.540282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205136.540283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205136.540286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205136.540288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205136.540435:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205136.540442:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205136.540445:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205136.540450:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205136.540455:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205136.540459:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205136.540462:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205136.540466:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205136.540469:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205136.540909:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205136.540913:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205136.540916:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205136.541106:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205136.541111:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205136.541112:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205136.541115:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205136.541118:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205136.541120:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205136.541123:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205136.541125:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205136.541127:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205137.052234:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321495815, at 4321495872 00000100:00000040:3.0:1788205137.052234:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321495815, at 4321495872 00000100:00000010:2.0:1788205137.052254:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc900. 00000100:00000010:3.0:1788205137.052257:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc540. 00000100:00000001:2.0:1788205137.564230:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205137.564230:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205137.564237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205137.564240:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205137.564241:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205137.564242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205137.564244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205137.564244:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205137.564253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205137.564253:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205137.564255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205137.564258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205137.564261:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205137.564262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205137.564264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205137.564268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205137.564268:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205137.564268:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205137.564275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205137.564275:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205137.564278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205137.564281:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205137.564281:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205137.564282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205137.564288:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205137.564289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205137.564360:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205137.564366:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205137.564370:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205138.588197:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205138.588200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205138.588201:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205138.588206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205138.588207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205138.588212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205138.588215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205138.588218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205138.588218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205138.588219:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205138.588226:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205138.588228:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205138.588229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205138.588237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205138.588238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205138.588238:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205138.588242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205138.588247:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205138.588248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205138.588248:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205138.588252:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205138.588254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205138.588262:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205138.588264:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205138.588270:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205138.588273:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205138.588358:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205138.588362:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205138.588365:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205139.612153:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205139.612165:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205139.612166:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205139.612167:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205139.612172:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205139.612176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205139.612176:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205139.612179:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205139.612183:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205139.612184:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205139.612190:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205139.612190:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205139.612196:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205139.612197:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205139.612201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205139.612201:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205139.612206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205139.612207:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205139.612317:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205139.612324:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205139.612326:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205139.612330:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205139.612335:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205139.612338:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205139.612341:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205139.612343:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205139.612348:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205139.612350:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205139.612353:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205139.612355:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205139.612359:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205139.612360:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205139.612757:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205139.612763:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205139.612767:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205140.636222:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205140.636228:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205140.636231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205140.636236:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.636238:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205140.636240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205140.636242:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205140.636245:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205140.636247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205140.636250:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205140.636257:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205140.636257:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205140.636260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205140.636263:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205140.636264:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205140.636265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.636270:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205140.636276:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205140.636278:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205140.636279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205140.636282:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205140.636283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.636286:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.636290:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205140.636293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205140.636297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205140.636447:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205140.636452:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205140.636456:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205140.828256:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205140.828264:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205140.828266:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205140.828269:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205140.828273:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205140.828278:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26827 00000400:00000200:2.0:1788205140.828287:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205140.828295:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205140.828297:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205140.828301:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828304:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205140.828306:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828308:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205140.828324:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205140.828327:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205140.828330:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205140.828333:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205140.828336:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205140.828338:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205140.828341:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828343:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205140.828359:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828364:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205140.828367:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205140.828371:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828374:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205140.828377:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205140.828380:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205140.828383:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.828385:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205140.828388:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205140.828397:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000020:00000040:2.0:1788205140.828401:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205140.828405:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205140.828407:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.828410:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205140.828413:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205140.828417:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205140.828423:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788205140.828429:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205140.828437:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205140.828447:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205140.828451:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205140.828456:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205140.828487:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205140.828491:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205140.828494:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205140.828496:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205140.828498:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205140.828500:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205140.828503:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26827 00000400:00000200:2.0:1788205140.828507:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205140.828510:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205140.828513:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205140.828517:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828519:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205140.828522:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828524:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205140.828526:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205140.828528:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000100:00000001:3.0:1788205140.828529:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788205140.828530:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205140.828533:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:3.0:1788205140.828535:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788205140.828536:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205140.828540:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788205140.828541:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788205140.828548:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205140.828550:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205140.828551:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205140.828555:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828559:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205140.828564:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:3.0:1788205140.828568:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788205140.828570:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205140.828572:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.828575:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.828578:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205140.828583:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205140.828586:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205140.828588:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788205140.828593:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543515904:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788205140.828595:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205140.828599:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205140.828600:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.828604:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205140.828606:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1788205140.828608:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205140.828611:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788205140.828612:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.828616:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788205140.828617:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205140.828623:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 02000000:00000010:3.0:1788205140.828623:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 00000020:00000040:2.0:1788205140.828626:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205140.828628:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205140.828631:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205140.828633:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.828636:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1788205140.828637:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788205140.828638:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205140.828642:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205140.828644:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 02000000:00000010:2.0:1788205140.828647:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788205140.828652:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205140.828658:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788205140.828663:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205140.828674:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543515904, portal 25 00000100:00000001:2.0:1788205140.828677:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205140.828677:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205140.828680:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331918464 : -109424377633152 : ffff9c7aa6d76680) 00000100:00000001:2.0:1788205140.828687:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205140.828689:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205140.828695:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.828706:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788205140.828711:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205140.828714:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205140.828720:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:3.0:1788205140.828721:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828722:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205140.828726:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788205140.828727:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000001:2.0:1788205140.828729:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788205140.828734:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543515904, offset 0 00000100:00080000:2.0:1788205140.828735:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26827 00000400:00000200:2.0:1788205140.828742:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.828746:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:2.0:1788205140.828749:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205140.828755:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205140.828758:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788205140.828761:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828765:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205140.828768:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000040:2.0:1788205140.828771:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828775:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205140.828777:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1788205140.828780:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205140.828782:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788205140.828784:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205140.828786:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788205140.828788:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788205140.828790:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205140.828792:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000040:2.0:1788205140.828796:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828799:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205140.828799:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1788205140.828803:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828806:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205140.828808:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000040:2.0:1788205140.828809:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788205140.828814:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828817:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205140.828820:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205140.828820:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471379:471379:256:4294967295] 192.168.201.118@tcp LPNI seq info [471238:471238:8:4294967295] 02000000:00000001:2.0:1788205140.828824:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205140.828827:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.828830:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205140.828831:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00100000:2.0:1788205140.828833:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205140.828837:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.828841:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205140.828846:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000010:2.0:1788205140.828848:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788205140.828852:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000800:00000200:3.0:1788205140.828855:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:2.0:1788205140.828857:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205140.828860:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.828862:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:3.0:1788205140.828863:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 02000000:00000001:2.0:1788205140.828864:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205140.828867:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205140.828874:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000800:00000200:3.0:1788205140.828874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205140.828880:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000001:2.0:1788205140.828881:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205140.828890:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.1:1788205140.828890:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205140.828897:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205140.828906:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828907:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:3.0:1788205140.828909:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000100:00000001:2.0:1788205140.828911:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205140.828917:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205140.828917:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205140.828927:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788205140.828933:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205140.828937:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000100:00080000:2.0:1788205140.828938:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788205140.828942:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00080000:2.0:1788205140.828944:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205140.828947:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788205140.828948:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1788205140.828949:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205140.828955:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788205140.828956:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205140.828960:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26827 00000400:00000200:2.0:1788205140.828971:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205140.828974:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1788205140.828977:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205140.828982:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205140.828982:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205140.828989:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.828994:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205140.828999:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205140.829001:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.829002:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205140.829006:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1788205140.829007:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205140.829010:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788205140.829012:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788205140.829013:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205140.829014:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788205140.829019:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205140.829022:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788205140.829023:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788205140.829028:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205140.829029:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321500933 00000100:00000001:2.0:1788205140.829032:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205140.829036:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000010:3.0:1788205140.829037:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc540. 00000100:00000001:2.0:1788205140.829038:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205140.829043:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788205140.829047:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205140.829047:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1788205140.829053:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205140.829055:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205140.829058:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321500933 02000000:00000001:2.0:1788205140.829059:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205140.829063:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205140.829065:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc120. 00000100:00000001:2.0:1788205140.829066:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205140.829069:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:3.0:1788205140.829071:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000008:00000001:2.0:1788205140.829074:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829077:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205140.829078:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829085:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205140.829086:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000100:00000001:3.0:1788205140.829088:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205140.829126:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205140/real 1788205140] req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205140.829136:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205140.829138:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205140.829140:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000020:00000040:2.0:1788205140.829142:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000400:00000200:3.0:1788205140.829144:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 02000000:00000001:2.0:1788205140.829148:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000010:3.0:1788205140.829148:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 02000000:00000001:2.0:1788205140.829151:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829153:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788205140.829155:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205140.829159:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:3.0:1788205140.829160:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205140.829162:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205140.829169:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788205140.829176:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205140.829178:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205140.829183:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205140.829197:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205140.829200:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205140.829201:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205140.829204:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829206:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.829210:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205140.829211:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205140.829219:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205140.829224:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205140.829229:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205140.829235:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205140.829236:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205140.829239:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205140.829243:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1788205140.829244:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205140.829248:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205140.829253:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26827 00000100:00000001:3.0:1788205140.829259:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1788205140.829261:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205140.829266:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788205140.829267:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205140.829270:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788205140.829272:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205140.829274:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205140.829277:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205140.829279:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205140.829282:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205140.829283:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1788205140.829286:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.829291:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00080000:3.0:1788205140.829291:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1788205140.829295:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205140.829298:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00080000:3.0:1788205140.829300:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000020:00000001:2.0:1788205140.829301:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205140.829304:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:3.0:1788205140.829306:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:2.0:1788205140.829308:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788205140.829313:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788205140.829316:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.829318:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205140.829319:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205140.829323:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788205140.829324:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205140.829326:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788205140.829332:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205140.829339:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205140.829342:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205140.829358:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205140.829361:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.829364:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:3.0:1788205140.829364:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543515904:192.168.201.118@tcp:250:kworker.0 00000008:00100000:2.0:1788205140.829367:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788205140.829372:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829373:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205140.829374:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829378:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000010:2.0:1788205140.829380:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000100:00000040:3.0:1788205140.829384:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76680 x1875055543515904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205140.829385:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205140.829390:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205140.829393:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205140.829397:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205140.829400:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205140.829400:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788205140.829405:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.829405:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1788205140.829410:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 02000000:00000010:3.0:1788205140.829410:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 00000100:00000001:2.0:1788205140.829416:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.829417:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205140.829421:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205140.829425:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000100:00100000:2.0:1788205140.829426:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1788205140.829430:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205140.829434:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788205140.829438:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788205140.829441:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788205140.829442:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.829446:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:2.0:1788205140.829448:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788205140.829450:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76680. 00000100:00000001:3.0:1788205140.829456:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205140.829459:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205140.829462:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829463:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205140.829469:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26837) 00000100:00000001:3.0:1788205140.829473:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205140.829476:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829479:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205140.829480:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205140.829482:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205140.829486:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829494:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205140.829495:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.829497:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205140.829499:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205140.829501:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205140.829504:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516032:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205140.829507:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205140.829509:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205140.829511:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.829512:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205140.829515:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205140.829517:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205140.829521:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205140.829524:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205140.829527:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205140.829530:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543516032, portal 10 00000100:00000001:3.0:1788205140.829532:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205140.829534:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331915776 : -109424377635840 : ffff9c7aa6d75c00) 00000100:00000040:3.0:1788205140.829539:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829545:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205140.829547:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205140.829550:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205140.829552:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543516032, offset 0 00000400:00000200:3.0:1788205140.829556:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205140.829560:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.829564:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205140.829567:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205140.829570:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205140.829572:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205140.829574:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205140.829577:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471380:471380:256:4294967295] 192.168.201.118@tcp LPNI seq info [471239:471239:8:4294967295] 00000400:00000200:3.0:1788205140.829581:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.829587:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205140.829591:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205140.829595:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205140.829598:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205140.829602:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205140.829606:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205140.829608:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205140.829611:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205140.829613:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205140.829616:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205140.829619:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205140.829622:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205140.829623:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205140.829626:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205140.829629:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829637:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205140.829639:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829645:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829647:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205140.829650:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205140.829653:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205140.829656:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205140.829658:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321499667 00000100:00000001:3.0:1788205140.829661:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321500933 00000100:00000001:3.0:1788205140.829663:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321499667 00000100:00000001:3.0:1788205140.829665:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321500933 00000100:00000001:3.0:1788205140.829667:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205140.829670:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829671:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205140.829675:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205140/real 1788205140] req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205140.829681:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205140.829682:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205140.829684:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205140.829686:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205140.829688:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205140.829690:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205140.829692:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205140.829700:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829706:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205140.829707:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.829708:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.829710:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205140.829713:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829718:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205140.829722:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829729:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205140.829731:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.829733:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.829734:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.829736:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205140.829738:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205140.829740:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205140.829744:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205140.829746:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205140.829750:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205140.829757:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516032:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205140.829761:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205140.829765:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829771:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205140.829773:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.829775:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205140.829777:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205140.829778:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205140.829781:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516160:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205140.829784:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205140.829786:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205140.829788:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.829789:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205140.829792:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205140.829794:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205140.829797:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205140.829799:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205140.829803:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205140.829805:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543516160, portal 10 00000100:00000001:3.0:1788205140.829807:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205140.829808:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922048 : -109424377629568 : ffff9c7aa6d77480) 00000100:00000040:3.0:1788205140.829813:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829820:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205140.829822:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205140.829824:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205140.829827:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543516160, offset 0 00000400:00000200:3.0:1788205140.829830:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205140.829834:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.829838:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205140.829841:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205140.829843:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205140.829845:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205140.829848:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205140.829851:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471381:471381:256:4294967295] 192.168.201.118@tcp LPNI seq info [471240:471240:8:4294967295] 00000400:00000200:3.0:1788205140.829855:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.829861:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205140.829865:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205140.829868:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205140.829872:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205140.829874:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205140.829878:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205140.829881:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205140.829885:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205140.829886:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205140.829889:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205140.829892:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205140.829895:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205140.829897:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205140.829899:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205140.829902:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829910:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205140.829913:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829920:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829921:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205140.829924:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205140.829926:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205140.829929:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205140.829931:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321499668 00000100:00000001:3.0:1788205140.829934:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321500933 00000100:00000001:3.0:1788205140.829936:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321499668 00000100:00000001:3.0:1788205140.829938:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321500933 00000100:00000001:3.0:1788205140.829940:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205140.829942:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.829943:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205140.829946:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205140/real 1788205140] req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205140.829953:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205140.829955:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205140.829956:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205140.829959:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205140.829960:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205140.829962:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205140.829965:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205140.829973:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829978:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205140.829980:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.829981:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.829982:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205140.829986:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.829992:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205140.829996:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830003:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205140.830005:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830006:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.830008:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830009:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205140.830011:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205140.830013:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205140.830016:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205140.830018:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205140.830021:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205140.830029:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516160:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205140.830033:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205140.830036:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830042:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205140.830043:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.830045:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205140.830047:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205140.830049:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205140.830051:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516288:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205140.830054:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205140.830055:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205140.830057:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.830058:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205140.830061:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205140.830063:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205140.830066:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205140.830068:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205140.830071:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205140.830074:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543516288, portal 4 00000100:00000001:3.0:1788205140.830075:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205140.830077:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331914880 : -109424377636736 : ffff9c7aa6d75880) 00000100:00000040:3.0:1788205140.830082:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830088:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205140.830107:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205140.830110:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205140.830112:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543516288, offset 0 00000400:00000200:3.0:1788205140.830115:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205140.830119:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.830122:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205140.830125:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205140.830127:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205140.830129:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205140.830132:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205140.830135:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471382:471382:256:4294967295] 192.168.201.118@tcp LPNI seq info [471241:471241:8:4294967295] 00000400:00000200:3.0:1788205140.830139:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.830144:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205140.830148:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205140.830151:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205140.830154:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205140.830157:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205140.830161:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205140.830164:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205140.830167:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205140.830169:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205140.830172:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205140.830175:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205140.830177:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205140.830179:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205140.830182:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205140.830185:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830193:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205140.830196:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830203:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.830205:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205140.830209:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205140.830211:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205140.830214:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205140.830216:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321499678 00000100:00000001:3.0:1788205140.830218:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321500933 00000100:00000001:3.0:1788205140.830220:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321499678 00000100:00000001:3.0:1788205140.830222:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321500933 00000100:00000001:3.0:1788205140.830224:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205140.830226:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.830228:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205140.830231:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205140/real 1788205140] req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205140.830238:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205140.830239:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205140.830241:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205140.830243:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205140.830245:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205140.830246:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205140.830249:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205140.830258:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830264:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205140.830265:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830267:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.830268:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205140.830272:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830278:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205140.830281:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830288:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205140.830290:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830291:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.830292:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830294:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205140.830296:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205140.830298:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205140.830300:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205140.830303:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205140.830306:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205140.830314:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516288:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205140.830317:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205140.830320:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830326:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205140.830327:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.830330:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205140.830331:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205140.830333:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205140.830336:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516416:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205140.830339:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205140.830341:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205140.830342:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205140.830344:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205140.830360:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205140.830362:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205140.830366:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205140.830368:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205140.830371:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205140.830373:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543516416, portal 4 00000100:00000001:3.0:1788205140.830375:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205140.830377:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331919360 : -109424377632256 : ffff9c7aa6d76a00) 00000100:00000040:3.0:1788205140.830382:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830388:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205140.830390:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205140.830392:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205140.830394:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543516416, offset 0 00000400:00000200:3.0:1788205140.830398:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205140.830401:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.830404:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205140.830407:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205140.830409:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205140.830411:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205140.830413:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205140.830416:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471383:471383:256:4294967295] 192.168.201.118@tcp LPNI seq info [471242:471242:8:4294967295] 00000400:00000200:3.0:1788205140.830420:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205140.830427:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205140.830431:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205140.830434:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205140.830437:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205140.830440:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205140.830444:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205140.830447:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205140.830450:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205140.830452:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205140.830455:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205140.830458:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205140.830461:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205140.830462:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205140.830464:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205140.830468:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830475:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205140.830478:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205165 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830484:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.830486:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205140.830488:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205140.830491:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205140.830494:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205140.830495:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321499701 00000100:00000001:3.0:1788205140.830498:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321500933 00000100:00000001:3.0:1788205140.830500:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321499701 00000100:00000001:3.0:1788205140.830502:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321500933 00000100:00000001:3.0:1788205140.830504:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205140.830505:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.830507:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205140.830510:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205140/real 1788205140] req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205140.830518:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205140.830520:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205140.830521:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205140.830523:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205140.830525:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205140.830526:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205140.830529:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205140.830537:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830543:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205140.830544:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830546:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.830547:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205140.830551:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830557:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205140.830560:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830566:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205140.830568:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830570:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205140.830571:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830572:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205140.830574:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205140.830576:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205140.830578:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205140.830580:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205140.830584:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205140.830592:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516416:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205140.830595:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830597:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205140.830600:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75c00 x1875055543516032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830605:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205140.830607:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205140.830609:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205140.830611:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205140.830612:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205140.830614:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205140.830616:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205140.830617:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:3.0:1788205140.830619:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205140.830621:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205140.830622:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75c00. 00000100:00000001:3.0:1788205140.830625:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205140.830626:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830627:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205140.830630:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77480 x1875055543516160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830636:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205140.830637:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205140.830638:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205140.830641:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205140.830642:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205140.830643:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205140.830645:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205140.830646:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:3.0:1788205140.830648:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205140.830650:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205140.830651:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77480. 00000100:00000001:3.0:1788205140.830654:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205140.830655:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830656:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205140.830659:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75880 x1875055543516288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830665:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205140.830666:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205140.830667:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205140.830669:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205140.830670:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205140.830671:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205140.830672:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205140.830674:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:3.0:1788205140.830676:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205140.830677:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205140.830678:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75880. 00000100:00000001:3.0:1788205140.830680:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205140.830681:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830682:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205140.830685:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76a00 x1875055543516416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205165 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205140.830691:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205140.830693:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205140.830694:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205140.830696:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205140.830697:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205140.830698:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205140.830699:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205140.830701:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:3.0:1788205140.830702:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205140.830703:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205140.830704:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76a00. 00000100:00000001:3.0:1788205140.830706:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205140.830707:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830709:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205140.830713:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205140.830714:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205140.830716:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205140.830717:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205141.660192:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205141.660193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205141.660199:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205141.660201:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205141.660206:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205141.660207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205141.660207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205141.660213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205141.660218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205141.660218:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788205141.660219:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205141.660225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205141.660225:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205141.660226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205141.660231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205141.660241:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1788205141.660241:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205141.660242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205141.660246:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205141.660247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.660253:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205141.660254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205141.660258:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.660263:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205141.660266:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205141.660270:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205141.660308:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205141.660314:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205141.660318:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205142.172246:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321500933, at 4321500992 00000100:00000040:3.0:1788205142.172246:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321500933, at 4321500992 00000100:00000010:2.0:1788205142.172267:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc540. 00000100:00000010:3.0:1788205142.172274:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc120. 00000100:00000001:3.0:1788205142.684226:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205142.684231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205142.684235:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205142.684237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205142.684239:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205142.684248:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205142.684249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205142.684254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205142.684254:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205142.684258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205142.684260:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205142.684262:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205142.684267:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205142.684268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205142.684271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205142.684272:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205142.684277:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205142.684372:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205142.684375:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205142.684378:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205142.684385:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205142.684391:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205142.684393:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205142.684396:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205142.684398:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205142.684400:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205142.684403:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205142.684406:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205142.684408:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205143.708236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205143.708244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205143.708249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205143.708250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205143.708252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205143.708253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205143.708255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205143.708256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205143.708350:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205143.708354:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205143.708357:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205143.708391:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205143.708398:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205143.708400:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205143.708405:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205143.708409:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205143.708413:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205143.708416:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205143.708420:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205143.708423:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205143.708588:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205143.708593:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205143.708595:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205143.708598:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205143.708600:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205143.708602:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205143.708605:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205143.708607:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205143.708609:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.732228:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205144.732239:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205144.732242:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205144.732248:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205144.732255:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205144.732260:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205144.732261:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205144.732264:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205144.732268:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205144.732269:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205144.732272:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205144.732274:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205144.732276:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205144.732307:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205144.732310:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205144.732313:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205144.732316:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205144.732318:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733629:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205144.733635:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733642:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205144.733646:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205144.733648:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205144.733652:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733656:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205144.733659:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733661:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733663:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733667:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205144.733669:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205144.733670:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205144.733671:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733673:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205144.733674:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733676:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733678:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733681:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205144.733683:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205144.733684:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205144.733686:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733687:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205144.733688:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733691:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205144.733693:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205144.734340:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205144.734345:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205144.734348:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205145.756173:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205145.756177:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205145.756179:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205145.756180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205145.756182:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205145.756186:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205145.756187:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205145.756189:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205145.756191:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205145.756193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205145.756194:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205145.756195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205145.756197:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205145.756199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205145.756201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205145.756201:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205145.756206:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205145.756253:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205145.756256:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205145.756259:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205145.756491:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205145.756499:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205145.756501:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205145.756506:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205145.756510:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205145.756514:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.756517:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205145.756520:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.756522:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205145.948255:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205145.948287:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205145.948290:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205145.948293:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205145.948298:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205145.948306:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26832 00000400:00000200:0.0:1788205145.948315:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205145.948322:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205145.948325:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205145.948328:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948331:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205145.948333:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948336:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205145.948340:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205145.948342:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205145.948344:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205145.948348:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205145.948350:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205145.948353:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205145.948356:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948358:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205145.948360:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948363:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205145.948366:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205145.948369:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948371:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205145.948374:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205145.948377:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205145.948380:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.948390:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205145.948392:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205145.948405:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788205145.948409:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205145.948412:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205145.948414:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.948417:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205145.948420:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205145.948424:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205145.948431:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788205145.948438:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205145.948447:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205145.948455:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205145.948458:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205145.948461:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205145.948479:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205145.948482:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205145.948485:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205145.948487:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205145.948488:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205145.948490:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205145.948493:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26832 00000400:00000200:0.0:1788205145.948498:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205145.948501:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205145.948503:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205145.948506:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948509:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205145.948512:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948515:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205145.948517:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205145.948518:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205145.948520:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205145.948522:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205145.948524:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205145.948534:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788205145.948540:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948542:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205145.948544:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948546:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205145.948549:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:3.0:1788205145.948552:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788205145.948553:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948561:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.948563:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788205145.948564:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205145.948569:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788205145.948572:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205145.948576:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.948578:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:3.0:1788205145.948586:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788205145.948587:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205145.948597:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.948605:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205145.948614:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000100:00000001:3.0:1788205145.948615:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1788205145.948621:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000001:3.0:1788205145.948621:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205145.948628:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205145.948629:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205145.948631:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.948636:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205145.948636:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205145.948640:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205145.948642:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205145.948644:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205145.948651:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543516544:192.168.201.118@tcp:250:kworker.0 02000000:00000010:0.0:1788205145.948653:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000001:0.0:1788205145.948664:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.948664:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205145.948671:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1788205145.948674:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205145.948682:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205145.948684:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205145.948686:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205145.948687:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.948697:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205145.948708:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:0.0:1788205145.948712:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.948713:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205145.948720:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205145.948728:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205145.948729:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205145.948734:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205145.948738:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788205145.948738:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00000001:0.0:1788205145.948745:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:3.0:1788205145.948752:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00080000:0.0:1788205145.948755:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26832 00000400:00000200:0.0:1788205145.948764:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:3.0:1788205145.948766:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543516544, portal 25 00000400:00000200:0.0:1788205145.948770:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205145.948772:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788205145.948775:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205145.948776:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904116736 : -109419805434880 : ffff9c7bb75d9c00) 00000100:00000040:0.0:1788205145.948780:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948785:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205145.948790:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205145.948792:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948804:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205145.948808:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788205145.948811:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205145.948818:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000100:00000040:3.0:1788205145.948818:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000020:00000001:0.0:1788205145.948825:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:3.0:1788205145.948825:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000020:00000040:0.0:1788205145.948831:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000200:3.0:1788205145.948833:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543516544, offset 0 00000020:00000001:0.0:1788205145.948836:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205145.948845:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000020:00000001:0.0:1788205145.948847:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788205145.948854:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948857:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205145.948861:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:0.0:1788205145.948863:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948868:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205145.948870:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788205145.948875:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00080000:0.0:1788205145.948879:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948885:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205145.948888:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1788205145.948890:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205145.948895:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205145.948897:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000001:0.0:1788205145.948899:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.948904:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205145.948906:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000008:00100000:0.0:1788205145.948908:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788205145.948914:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000008:00000001:0.0:1788205145.948916:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.948920:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205145.948926:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471384:471384:256:4294967295] 192.168.201.118@tcp LPNI seq info [471243:471243:8:4294967295] 00000100:00000010:0.0:1788205145.948928:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788205145.948936:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000200:3.0:1788205145.948941:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:0.0:1788205145.948943:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205145.948946:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.948949:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205145.948951:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205145.948954:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205145.948961:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000010:0.0:1788205145.948965:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000800:00000200:3.0:1788205145.948972:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205145.948978:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205145.948981:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000100:00100000:0.0:1788205145.948988:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788205145.948998:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788205145.949009:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205145.949011:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205145.949014:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000040:0.0:1788205145.949023:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788205145.949031:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205145.949044:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:0.0:1788205145.949049:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788205145.949056:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00080000:0.0:1788205145.949058:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:3.0:1788205145.949061:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000100:00080000:0.0:1788205145.949064:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205145.949068:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205145.949070:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205145.949072:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205145.949079:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205145.949086:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205145.949088:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205145.949136:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1788205145.949142:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788205145.949143:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205145.949150:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26832 00000400:00000200:0.0:1788205145.949165:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205145.949176:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205145.949184:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205145.949187:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205145.949189:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.949193:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205145.949197:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205145.949198:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.949206:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205145.949215:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205145.949220:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205145.949224:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788205145.949225:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205145.949227:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00000001:3.0:1788205145.949230:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1788205145.949231:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788205145.949235:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788205145.949239:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.949240:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205145.949242:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205145.949243:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.949251:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205145.949252:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788205145.949254:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205145.949255:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205145.949256:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205145.949268:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205145.949271:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205145.949278:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205145.949283:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205145.949285:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321506054 00000008:00100000:0.0:1788205145.949289:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000010:3.0:1788205145.949295:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc120. 00000008:00000001:0.0:1788205145.949296:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.949302:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205145.949307:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 00000100:00000001:3.0:1788205145.949307:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000020:00000040:0.0:1788205145.949317:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205145.949320:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205145.949321:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321506054 02000000:00000001:0.0:1788205145.949322:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.949329:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:3.0:1788205145.949330:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcc60. 02000000:00000001:0.0:1788205145.949332:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205145.949335:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.949336:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000010:0.0:1788205145.949343:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:3.0:1788205145.949343:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205145.949350:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.949352:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.949357:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788205145.949362:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205145.949372:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788205145.949374:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205145.949376:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205145.949388:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205145.949406:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788205145.949409:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788205145.949410:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205145.949415:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000100:00080000:0.0:1788205145.949416:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205145.949421:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000100:00080000:0.0:1788205145.949422:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000010:3.0:1788205145.949427:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:0.0:1788205145.949428:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205145.949432:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205145.949435:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1788205145.949439:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788205145.949445:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788205145.949450:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26832 00000400:00000200:0.0:1788205145.949466:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:3.0:1788205145.949471:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205145.949477:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205145.949485:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205145.949495:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:0.0:1788205145.949496:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205145.949502:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205145.949503:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205145.949510:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1788205145.949511:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205145.949514:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205145.949516:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205145.949520:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205145.949523:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00100000:3.0:1788205145.949524:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788205145.949527:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205145.949531:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205145.949540:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205145.949550:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205145.949550:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1788205145.949564:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205145.949565:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205145.949568:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205145.949575:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205145.949582:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205145.949592:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:3.0:1788205145.949593:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788205145.949600:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205145.949601:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.949608:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788205145.949610:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.949611:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788205145.949619:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205145.949619:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205145.949626:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1788205145.949627:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788205145.949634:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1788205145.949635:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.949644:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205145.949649:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1788205145.949649:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000008:00000001:0.0:1788205145.949658:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.949658:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788205145.949666:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205145.949672:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788205145.949674:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788205145.949680:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205145.949692:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205145.949695:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205145.949704:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205145.949706:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1788205145.949709:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543516544:192.168.201.118@tcp:250:kworker.0 02000000:00000001:0.0:1788205145.949710:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205145.949719:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b400. 00000100:00000001:3.0:1788205145.949722:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205145.949725:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.949727:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.949738:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9c00 x1875055543516544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788205145.949740:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205145.949781:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205145.949788:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788205145.949792:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000010:3.0:1788205145.949794:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 00000100:00000001:0.0:1788205145.949798:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.949801:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205145.949806:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000040:0.0:1788205145.949807:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205145.949812:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205145.949819:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205145.949824:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788205145.949827:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.949832:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:0.0:1788205145.949836:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26842) 02000000:00000001:3.0:1788205145.949836:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205145.949840:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9c00. 00000100:00000001:3.0:1788205145.949846:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205145.949847:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.949850:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.949859:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205145.949862:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.949866:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205145.949868:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205145.949870:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205145.949876:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.949883:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205145.949885:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.949887:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205145.949889:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205145.949892:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205145.949895:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516672:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205145.949898:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205145.949901:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205145.949903:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.949906:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205145.949909:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205145.949911:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205145.949917:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205145.949921:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205145.949924:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205145.949927:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543516672, portal 10 00000100:00000001:3.0:1788205145.949929:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205145.949931:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904120320 : -109419805431296 : ffff9c7bb75daa00) 00000100:00000040:3.0:1788205145.949938:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.949946:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205145.949948:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205145.949950:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205145.949952:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543516672, offset 0 00000400:00000200:3.0:1788205145.949957:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205145.949961:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205145.949965:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205145.949968:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205145.949970:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205145.949972:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205145.949975:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205145.949978:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471385:471385:256:4294967295] 192.168.201.118@tcp LPNI seq info [471244:471244:8:4294967295] 00000400:00000200:3.0:1788205145.949982:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205145.949989:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205145.949994:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205145.949997:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205145.950002:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205145.950005:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205145.950009:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205145.950012:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205145.950017:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205145.950019:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205145.950022:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205145.950025:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205145.950028:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205145.950030:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205145.950032:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205145.950036:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950043:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.950046:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950058:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.950060:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205145.950063:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205145.950067:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205145.950071:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205145.950073:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321504787 00000100:00000001:3.0:1788205145.950076:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321506054 00000100:00000001:3.0:1788205145.950078:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321504787 00000100:00000001:3.0:1788205145.950080:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321506054 00000100:00000001:3.0:1788205145.950083:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205145.950085:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.950087:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205145.950111:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205145.950119:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205145.950121:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205145.950123:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205145.950125:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205145.950127:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205145.950130:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205145.950134:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205145.950142:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950148:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205145.950150:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950151:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205145.950153:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205145.950157:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950164:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205145.950168:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950175:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205145.950177:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950179:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205145.950180:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950182:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205145.950185:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205145.950187:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.950190:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205145.950192:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205145.950195:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.950202:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516672:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205145.950206:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205145.950210:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950217:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205145.950218:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.950220:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205145.950222:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205145.950224:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205145.950227:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516800:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205145.950230:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205145.950233:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205145.950235:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.950237:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205145.950240:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205145.950242:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205145.950246:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205145.950248:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205145.950251:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205145.950254:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543516800, portal 10 00000100:00000001:3.0:1788205145.950256:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205145.950270:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904115840 : -109419805435776 : ffff9c7bb75d9880) 00000100:00000040:3.0:1788205145.950277:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950283:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205145.950286:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205145.950289:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205145.950291:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543516800, offset 0 00000400:00000200:3.0:1788205145.950295:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205145.950299:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205145.950303:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205145.950306:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205145.950308:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205145.950311:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205145.950313:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205145.950316:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471386:471386:256:4294967295] 192.168.201.118@tcp LPNI seq info [471245:471245:8:4294967295] 00000400:00000200:3.0:1788205145.950320:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205145.950325:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205145.950330:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205145.950334:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205145.950338:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205145.950340:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205145.950344:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205145.950347:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205145.950350:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205145.950353:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205145.950355:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205145.950359:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205145.950362:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205145.950363:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205145.950365:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205145.950369:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950376:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.950379:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950385:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.950386:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205145.950389:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205145.950392:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205145.950396:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205145.950398:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321504789 00000100:00000001:3.0:1788205145.950400:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321506054 00000100:00000001:3.0:1788205145.950402:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321504789 00000100:00000001:3.0:1788205145.950404:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321506054 00000100:00000001:3.0:1788205145.950406:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205145.950408:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.950410:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205145.950413:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205145.950421:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205145.950422:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205145.950423:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205145.950426:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205145.950428:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205145.950431:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205145.950434:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205145.950441:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950446:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205145.950447:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950449:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205145.950450:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205145.950453:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950459:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205145.950462:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950468:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205145.950470:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950472:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205145.950473:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950475:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205145.950478:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205145.950480:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.950483:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205145.950486:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205145.950489:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.950496:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516800:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205145.950501:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205145.950505:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950511:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205145.950513:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.950515:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205145.950517:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205145.950519:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205145.950521:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516928:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205145.950525:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205145.950527:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205145.950528:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.950530:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205145.950533:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205145.950535:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205145.950539:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205145.950541:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205145.950545:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205145.950548:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543516928, portal 4 00000100:00000001:3.0:1788205145.950549:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205145.950551:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904124800 : -109419805426816 : ffff9c7bb75dbb80) 00000100:00000040:3.0:1788205145.950556:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950562:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205145.950564:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205145.950566:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205145.950569:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543516928, offset 0 00000400:00000200:3.0:1788205145.950573:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205145.950576:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205145.950579:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205145.950581:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205145.950584:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205145.950586:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205145.950588:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205145.950591:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471387:471387:256:4294967295] 192.168.201.118@tcp LPNI seq info [471246:471246:8:4294967295] 00000400:00000200:3.0:1788205145.950595:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205145.950600:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205145.950604:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205145.950607:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205145.950611:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205145.950614:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205145.950617:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205145.950620:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205145.950623:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205145.950625:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205145.950627:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205145.950630:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205145.950633:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205145.950635:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205145.950637:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205145.950640:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950647:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.950650:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950656:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.950657:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205145.950660:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205145.950663:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205145.950666:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205145.950667:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321504798 00000100:00000001:3.0:1788205145.950670:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321506054 00000100:00000001:3.0:1788205145.950672:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321504798 00000100:00000001:3.0:1788205145.950674:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321506054 00000100:00000001:3.0:1788205145.950676:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205145.950678:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.950680:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205145.950683:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205145.950690:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205145.950691:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205145.950693:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205145.950695:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205145.950696:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205145.950698:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205145.950701:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205145.950709:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950715:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205145.950717:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950718:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205145.950719:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205145.950723:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950729:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205145.950733:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950739:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205145.950741:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950743:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205145.950744:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950746:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205145.950748:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205145.950751:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.950754:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205145.950756:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205145.950760:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.950767:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543516928:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205145.950771:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205145.950774:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950781:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205145.950782:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.950785:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205145.950786:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205145.950788:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205145.950791:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517056:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205145.950794:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205145.950796:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205145.950798:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205145.950800:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205145.950803:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205145.950805:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205145.950808:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205145.950811:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205145.950814:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205145.950817:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543517056, portal 4 00000100:00000001:3.0:1788205145.950819:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205145.950820:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904119424 : -109419805432192 : ffff9c7bb75da680) 00000100:00000040:3.0:1788205145.950826:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950832:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205145.950835:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205145.950837:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205145.950839:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543517056, offset 0 00000400:00000200:3.0:1788205145.950843:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205145.950847:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205145.950850:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205145.950852:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205145.950855:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205145.950857:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205145.950859:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205145.950862:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471388:471388:256:4294967295] 192.168.201.118@tcp LPNI seq info [471247:471247:8:4294967295] 00000400:00000200:3.0:1788205145.950865:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205145.950871:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205145.950875:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205145.950878:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205145.950882:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205145.950885:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205145.950888:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205145.950891:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205145.950895:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205145.950897:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205145.950900:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205145.950904:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205145.950907:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205145.950908:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205145.950910:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205145.950914:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950921:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.950924:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205170 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950930:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.950932:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205145.950935:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205145.950938:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205145.950941:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205145.950943:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321504821 00000100:00000001:3.0:1788205145.950945:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321506054 00000100:00000001:3.0:1788205145.950948:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321504821 00000100:00000001:3.0:1788205145.950950:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321506054 00000100:00000001:3.0:1788205145.950952:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205145.950953:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.950955:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205145.950958:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205145/real 1788205145] req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205145.950967:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205145.950968:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205145.950970:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205145.950972:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205145.950974:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205145.950976:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205145.950979:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205145.950987:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.950993:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205145.950995:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.950996:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205145.950998:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205145.951001:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.951007:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205145.951011:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.951018:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205145.951020:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.951022:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205145.951023:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.951025:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205145.951026:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205145.951029:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205145.951032:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205145.951035:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205145.951038:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205145.951045:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517056:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205145.951049:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.951051:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.951054:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75daa00 x1875055543516672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.951060:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205145.951062:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205145.951064:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205145.951067:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205145.951068:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205145.951070:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205145.951072:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205145.951073:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a400. 02000000:00000001:3.0:1788205145.951076:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205145.951077:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205145.951079:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75daa00. 00000100:00000001:3.0:1788205145.951081:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205145.951082:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.951085:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.951088:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9880 x1875055543516800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.951112:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205145.951113:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205145.951114:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205145.951117:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205145.951118:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205145.951119:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205145.951121:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205145.951123:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2bc00. 02000000:00000001:3.0:1788205145.951125:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205145.951126:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205145.951128:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9880. 00000100:00000001:3.0:1788205145.951130:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205145.951131:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.951133:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.951136:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75dbb80 x1875055543516928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.951142:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205145.951143:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205145.951144:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205145.951146:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205145.951147:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205145.951148:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205145.951150:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205145.951151:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29000. 02000000:00000001:3.0:1788205145.951153:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205145.951154:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205145.951155:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dbb80. 00000100:00000001:3.0:1788205145.951157:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205145.951158:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.951160:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205145.951163:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da680 x1875055543517056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205170 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205145.951174:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205145.951175:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205145.951177:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205145.951179:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205145.951180:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205145.951181:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205145.951183:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205145.951184:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b400. 02000000:00000001:3.0:1788205145.951186:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205145.951187:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205145.951189:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da680. 00000100:00000001:3.0:1788205145.951191:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205145.951192:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.951194:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205145.951197:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205145.951198:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205145.951200:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205145.951202:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205146.780232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205146.780236:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205146.780241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205146.780241:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205146.780255:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205146.780256:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205146.780259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205146.780261:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205146.780271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205146.780272:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205146.780278:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205146.780281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205146.780284:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205146.780285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205146.780288:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205146.780290:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205146.780293:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205146.780294:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205146.780296:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205146.780297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205146.780302:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205146.780305:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205146.780306:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205146.780307:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205146.780311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205146.780315:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205146.780561:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205146.780567:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205146.780570:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205147.292271:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321506054, at 4321506112 00000100:00000040:3.0:1788205147.292272:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321506054, at 4321506112 00000100:00000010:2.0:1788205147.292292:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc120. 00000100:00000010:3.0:1788205147.292296:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcc60. 00000100:00000001:3.0:1788205147.804210:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205147.804219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205147.804219:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205147.804222:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205147.804239:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205147.804240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205147.804244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205147.804246:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205147.804251:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205147.804253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205147.804261:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205147.804262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205147.804267:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205147.804270:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205147.804274:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205147.804275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205147.804281:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205147.804285:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205147.804349:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205147.804354:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205147.804360:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205147.804361:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205147.804364:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205147.804366:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205147.804369:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205147.804370:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205147.804430:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205147.804434:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205147.804436:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205148.828230:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205148.828231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205148.828235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205148.828242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205148.828243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205148.828244:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205148.828245:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205148.828250:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205148.828263:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205148.828263:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205148.828264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205148.828274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205148.828279:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205148.828279:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205148.828281:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205148.828288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205148.828289:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205148.828290:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205148.828295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205148.828295:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205148.828297:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205148.828299:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205148.828300:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205148.828302:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205148.828310:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205148.828314:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205148.828382:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205148.828388:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205148.828391:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205149.852230:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205149.852238:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205149.852241:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205149.852245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205149.852246:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205149.852251:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205149.852260:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205149.852261:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205149.852261:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205149.852268:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205149.852270:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205149.852270:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205149.852280:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205149.852280:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205149.852282:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205149.852290:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205149.852293:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.852293:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205149.852296:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.852298:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205149.852304:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205149.852304:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205149.852305:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205149.852312:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205149.852320:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205149.852322:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205149.852331:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205149.852333:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205149.852337:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205149.852338:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205149.852343:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205149.852344:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205149.852388:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205149.852393:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205149.852397:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205150.876231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205150.876231:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205150.876237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205150.876243:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205150.876244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205150.876244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205150.876247:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205150.876248:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205150.876256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205150.876256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205150.876256:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205150.876261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205150.876262:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205150.876263:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205150.876267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205150.876268:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205150.876268:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205150.876270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205150.876275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205150.876275:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205150.876277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205150.876279:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205150.876281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205150.876283:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205150.876285:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205150.876289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205150.876385:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205150.876389:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205150.876392:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205151.068238:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205151.068254:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205151.068256:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205151.068261:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205151.068266:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205151.068275:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26837 00000400:00000200:1.0:1788205151.068288:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205151.068296:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205151.068300:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205151.068303:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068307:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205151.068309:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068312:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205151.068316:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205151.068318:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205151.068322:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205151.068326:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205151.068329:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205151.068332:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205151.068335:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068337:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205151.068339:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068341:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205151.068344:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205151.068347:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068349:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205151.068351:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205151.068355:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205151.068358:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068359:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205151.068362:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205151.068372:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205151.068377:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205151.068382:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205151.068384:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068387:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205151.068391:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205151.068396:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205151.068407:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba000. 00000100:00000001:1.0:1788205151.068415:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205151.068425:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205151.068437:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205151.068443:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205151.068447:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205151.068467:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205151.068470:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205151.068473:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205151.068474:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205151.068476:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205151.068479:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205151.068482:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26837 00000400:00000200:1.0:1788205151.068486:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205151.068488:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205151.068490:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205151.068493:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068495:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205151.068497:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068499:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205151.068501:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205151.068502:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205151.068504:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788205151.068506:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1788205151.068507:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205151.068509:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205151.068511:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205151.068511:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788205151.068514:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205151.068514:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205151.068516:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205151.068518:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068520:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788205151.068521:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205151.068522:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205151.068525:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068528:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205151.068530:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205151.068531:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.068533:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205151.068535:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205151.068537:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.068537:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788205151.068539:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205151.068540:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205151.068542:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788205151.068543:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788205151.068545:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543517184:192.168.201.118@tcp:250:kworker.0 00000008:00000001:1.0:1788205151.068553:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068554:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788205151.068558:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.068559:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1788205151.068561:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 02000000:00000001:3.0:1788205151.068562:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205151.068563:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205151.068565:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205151.068566:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205151.068567:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068568:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205151.068569:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:3.0:1788205151.068570:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205151.068571:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.068572:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205151.068576:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba400. 00000100:00000001:1.0:1788205151.068579:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205151.068579:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205151.068583:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000100:00100000:1.0:1788205151.068584:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205151.068590:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000010:3.0:1788205151.068591:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000001:1.0:1788205151.068592:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205151.068595:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205151.068599:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543517184, portal 25 00000100:00000001:3.0:1788205151.068602:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205151.068604:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205151.068609:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205151.068615:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068617:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788205151.068618:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1788205151.068619:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:1.0:1788205151.068622:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:3.0:1788205151.068622:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000001:1.0:1788205151.068623:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205151.068625:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:3.0:1788205151.068626:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543517184, offset 0 00000100:00000001:1.0:1788205151.068628:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205151.068630:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26837 00000400:00000200:1.0:1788205151.068633:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.068633:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205151.068636:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205151.068638:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205151.068640:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068642:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205151.068642:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:1.0:1788205151.068644:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068646:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205151.068648:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205151.068649:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:3.0:1788205151.068649:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:1.0:1788205151.068651:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205151.068653:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205151.068655:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205151.068655:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:1.0:1788205151.068657:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205151.068659:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205151.068660:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:1.0:1788205151.068661:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205151.068663:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205151.068664:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1788205151.068665:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205151.068667:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788205151.068667:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00080000:1.0:1788205151.068669:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068670:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205151.068672:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205151.068672:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471389:471389:256:4294967295] 192.168.201.118@tcp LPNI seq info [471248:471248:8:4294967295] 02000000:00000001:1.0:1788205151.068675:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205151.068677:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:1.0:1788205151.068678:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068679:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205151.068681:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205151.068683:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068685:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205151.068688:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000400:00000200:3.0:1788205151.068689:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000020:00000040:1.0:1788205151.068690:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205151.068693:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205151.068694:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068695:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:3.0:1788205151.068696:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:1.0:1788205151.068697:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205151.068699:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205151.068702:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 02000000:00000010:1.0:1788205151.068703:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bbc00. 00000100:00000001:1.0:1788205151.068706:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205151.068710:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788205151.068710:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205151.068714:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000100:00000001:1.0:1788205151.068717:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205151.068719:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788205151.068721:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:1.0:1788205151.068722:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.1:1788205151.068725:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205151.068729:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068730:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205151.068733:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:3.0:1788205151.068733:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000100:00080000:1.0:1788205151.068736:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205151.068737:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205151.068738:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205151.068740:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1788205151.068741:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205151.068744:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26837 00000400:00000200:3.0:1788205151.068745:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205151.068747:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205151.068749:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205151.068751:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205151.068751:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000100:00000040:1.0:1788205151.068754:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000010:3.0:1788205151.068754:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:1.0:1788205151.068756:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205151.068758:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068759:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205151.068759:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1788205151.068762:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000200:3.0:1788205151.068762:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205151.068763:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205151.068764:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205151.068766:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205151.068769:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205151.068770:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:3.0:1788205151.068772:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205151.068773:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205151.068774:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205151.068776:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205151.068776:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205151.068778:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205151.068781:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205151.068783:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205151.068784:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068786:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068786:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1788205151.068787:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205151.068789:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205151.068790:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068792:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205151.068792:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788205151.068794:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205151.068795:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205151.068796:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205151.068797:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068799:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321511173 00000100:00000010:1.0:1788205151.068800:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205151.068802:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205151.068804:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000010:3.0:1788205151.068804:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc360. 02000000:00000001:1.0:1788205151.068806:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068807:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205151.068808:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205151.068810:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068812:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 02000000:00000010:1.0:1788205151.068813:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb800. 00000100:00000001:1.0:1788205151.068816:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068819:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321511173 00000100:00100000:1.0:1788205151.068820:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:3.0:1788205151.068822:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc840. 00000100:00000001:3.0:1788205151.068825:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205151.068827:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205151.068828:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205151.068829:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068831:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205151.068833:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.068834:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205151.068838:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205151/real 1788205151] req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205151.068840:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205151.068843:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205151.068846:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205151.068847:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205151.068849:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205151.068852:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205151.068854:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26837 00000400:00000001:3.0:1788205151.068854:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205151.068856:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1788205151.068857:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205151.068858:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:1.0:1788205151.068859:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205151.068861:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205151.068861:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205151.068863:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000040:1.0:1788205151.068864:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205151.068865:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1788205151.068866:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205151.068868:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000200:3.0:1788205151.068869:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205151.068870:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205151.068872:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205151.068874:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205151.068875:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205151.068877:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000200:3.0:1788205151.068878:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205151.068879:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205151.068881:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205151.068884:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205151.068885:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205151.068886:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205151.068886:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205151.068888:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205151.068889:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205151.068890:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205151.068891:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1788205151.068892:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205151.068895:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00100000:3.0:1788205151.068895:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205151.068897:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205151.068899:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205151.068901:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205151.068903:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068903:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205151.068905:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205151.068907:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000040:3.0:1788205151.068907:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1788205151.068909:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205151.068910:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205151.068913:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205151.068914:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:1.0:1788205151.068915:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205151.068918:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205151.068918:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788205151.068919:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068920:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205151.068921:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205151.068921:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788205151.068922:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205151.068923:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1788205151.068924:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068926:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:1.0:1788205151.068927:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9000. 00000100:00080000:3.0:1788205151.068929:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788205151.068930:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205151.068933:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00100000:1.0:1788205151.068934:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205151.068936:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205151.068940:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205151.068941:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205151.068942:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205151.068946:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205151.068950:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543517184:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205151.068953:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.068955:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.068957:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205151.068958:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26847) 00000100:00000040:3.0:1788205151.068959:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543517184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.068965:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205151.068968:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205151.068970:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205151.068974:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205151.068976:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205151.068977:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205151.068980:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205151.068983:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba000. 02000000:00000001:3.0:1788205151.068987:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205151.068989:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205151.068991:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205151.068994:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205151.068995:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.068997:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069005:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205151.069007:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069008:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205151.069009:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205151.069010:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205151.069013:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069018:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.069019:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.069020:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205151.069021:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205151.069023:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205151.069024:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205151.069027:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205151.069028:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205151.069029:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.069030:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205151.069033:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205151.069034:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205151.069037:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205151.069038:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205151.069041:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205151.069043:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543517312, portal 10 00000100:00000001:3.0:1788205151.069045:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205151.069046:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788205151.069051:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069055:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205151.069056:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205151.069058:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205151.069059:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543517312, offset 0 00000400:00000200:3.0:1788205151.069062:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205151.069064:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.069066:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205151.069068:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205151.069070:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205151.069071:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205151.069073:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205151.069075:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471390:471390:256:4294967295] 192.168.201.118@tcp LPNI seq info [471249:471249:8:4294967295] 00000400:00000200:3.0:1788205151.069078:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.069082:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205151.069084:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205151.069086:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205151.069118:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205151.069121:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205151.069124:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205151.069125:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205151.069128:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205151.069129:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205151.069131:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205151.069134:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205151.069136:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205151.069137:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205151.069139:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205151.069141:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069146:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205151.069148:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069152:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069153:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205151.069155:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205151.069158:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205151.069160:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205151.069161:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321509907 00000100:00000001:3.0:1788205151.069192:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321511173 00000100:00000001:3.0:1788205151.069194:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321509907 00000100:00000001:3.0:1788205151.069196:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321511173 00000100:00000001:3.0:1788205151.069197:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205151.069198:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069200:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205151.069202:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205151/real 1788205151] req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205151.069207:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205151.069208:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205151.069210:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205151.069212:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205151.069212:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205151.069214:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205151.069216:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205151.069221:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069225:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205151.069226:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069227:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205151.069227:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205151.069229:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069233:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205151.069235:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069239:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205151.069241:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069242:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205151.069242:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069243:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205151.069245:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205151.069246:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205151.069248:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205151.069249:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205151.069251:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205151.069256:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205151.069259:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205151.069261:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069264:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.069265:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.069266:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205151.069267:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205151.069268:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205151.069270:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205151.069272:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205151.069273:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205151.069274:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.069275:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205151.069278:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205151.069279:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205151.069281:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205151.069283:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205151.069284:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205151.069286:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543517440, portal 10 00000100:00000001:3.0:1788205151.069287:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205151.069288:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205151.069292:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069296:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205151.069297:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205151.069299:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205151.069300:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543517440, offset 0 00000400:00000200:3.0:1788205151.069302:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205151.069304:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.069306:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205151.069308:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205151.069310:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205151.069311:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205151.069313:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205151.069315:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471391:471391:256:4294967295] 192.168.201.118@tcp LPNI seq info [471250:471250:8:4294967295] 00000400:00000200:3.0:1788205151.069317:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.069321:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205151.069323:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205151.069325:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205151.069327:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205151.069329:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205151.069332:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205151.069333:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205151.069336:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205151.069337:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205151.069339:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205151.069341:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205151.069343:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205151.069344:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205151.069345:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205151.069347:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069352:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205151.069353:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069357:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069358:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205151.069360:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205151.069361:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205151.069363:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205151.069365:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321509908 00000100:00000001:3.0:1788205151.069366:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321511173 00000100:00000001:3.0:1788205151.069368:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321509908 00000100:00000001:3.0:1788205151.069370:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321511173 00000100:00000001:3.0:1788205151.069371:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205151.069372:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069373:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205151.069375:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205151/real 1788205151] req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205151.069379:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205151.069380:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205151.069381:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205151.069382:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205151.069383:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205151.069384:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205151.069385:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205151.069390:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069394:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205151.069395:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069396:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205151.069396:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205151.069398:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069402:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205151.069403:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069407:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205151.069408:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069409:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205151.069409:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069411:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205151.069413:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205151.069414:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205151.069416:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205151.069418:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205151.069420:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205151.069424:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205151.069425:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205151.069427:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069430:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.069431:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.069433:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205151.069434:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205151.069435:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205151.069437:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205151.069439:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205151.069440:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205151.069440:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.069441:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205151.069443:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205151.069444:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205151.069446:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205151.069447:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205151.069449:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205151.069451:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543517568, portal 4 00000100:00000001:3.0:1788205151.069452:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205151.069453:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205151.069457:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069461:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205151.069463:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205151.069464:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205151.069465:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543517568, offset 0 00000400:00000200:3.0:1788205151.069467:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205151.069469:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.069471:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205151.069472:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205151.069473:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205151.069475:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205151.069476:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205151.069478:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471392:471392:256:4294967295] 192.168.201.118@tcp LPNI seq info [471251:471251:8:4294967295] 00000400:00000200:3.0:1788205151.069480:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.069484:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205151.069487:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205151.069488:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205151.069490:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205151.069491:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205151.069493:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205151.069495:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205151.069497:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205151.069498:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205151.069500:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205151.069502:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205151.069504:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205151.069505:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205151.069506:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205151.069508:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069512:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205151.069514:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069521:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069522:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205151.069523:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205151.069525:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205151.069527:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205151.069528:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321509917 00000100:00000001:3.0:1788205151.069529:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321511173 00000100:00000001:3.0:1788205151.069531:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321509917 00000100:00000001:3.0:1788205151.069532:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321511173 00000100:00000001:3.0:1788205151.069533:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205151.069534:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069535:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205151.069537:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205151/real 1788205151] req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205151.069541:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205151.069541:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205151.069542:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205151.069543:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205151.069544:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205151.069545:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205151.069547:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205151.069552:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069555:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205151.069556:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069557:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205151.069557:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205151.069559:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069564:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205151.069566:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069570:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205151.069571:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069572:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205151.069572:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069573:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205151.069574:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205151.069575:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205151.069577:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205151.069578:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205151.069580:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205151.069584:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205151.069585:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205151.069587:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069590:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.069591:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.069592:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205151.069593:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205151.069594:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205151.069595:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205151.069597:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205151.069598:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205151.069599:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205151.069600:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205151.069602:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205151.069603:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205151.069605:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205151.069606:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205151.069608:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205151.069610:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543517696, portal 4 00000100:00000001:3.0:1788205151.069611:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205151.069612:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205151.069615:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069618:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205151.069619:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205151.069621:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205151.069622:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543517696, offset 0 00000400:00000200:3.0:1788205151.069625:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205151.069626:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.069628:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205151.069629:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205151.069631:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205151.069632:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205151.069633:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205151.069635:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471393:471393:256:4294967295] 192.168.201.118@tcp LPNI seq info [471252:471252:8:4294967295] 00000400:00000200:3.0:1788205151.069638:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205151.069641:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205151.069643:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205151.069644:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205151.069646:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205151.069648:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205151.069650:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205151.069652:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205151.069654:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205151.069656:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205151.069657:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205151.069659:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205151.069661:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205151.069662:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205151.069663:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205151.069665:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069669:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205151.069670:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205176 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069674:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069675:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205151.069677:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205151.069678:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205151.069679:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205151.069680:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321509940 00000100:00000001:3.0:1788205151.069681:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321511173 00000100:00000001:3.0:1788205151.069683:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321509940 00000100:00000001:3.0:1788205151.069684:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321511173 00000100:00000001:3.0:1788205151.069686:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205151.069687:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069688:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205151.069689:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205151/real 1788205151] req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205151.069695:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205151.069696:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205151.069697:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205151.069699:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205151.069700:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205151.069703:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205151.069706:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205151.069714:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069720:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205151.069721:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069723:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205151.069724:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205151.069728:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069734:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205151.069737:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069744:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205151.069746:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069748:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205151.069749:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069751:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205151.069753:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205151.069755:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205151.069758:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205151.069761:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205151.069764:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205151.069771:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205151.069773:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069775:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205151.069777:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543517312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069781:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205151.069782:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205151.069784:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205151.069786:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205151.069787:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205151.069788:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205151.069789:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205151.069790:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:3.0:1788205151.069792:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205151.069793:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205151.069794:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205151.069799:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205151.069800:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069802:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205151.069803:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543517440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069807:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205151.069808:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205151.069809:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205151.069810:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205151.069810:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205151.069811:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205151.069812:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205151.069813:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:3.0:1788205151.069814:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205151.069814:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205151.069815:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205151.069820:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205151.069821:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069821:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205151.069823:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543517568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069826:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205151.069827:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205151.069828:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205151.069830:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205151.069830:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205151.069831:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205151.069832:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205151.069833:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788205151.069834:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205151.069835:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205151.069835:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205151.069836:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205151.069837:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069838:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205151.069839:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543517696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205176 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205151.069842:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205151.069843:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205151.069844:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205151.069845:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205151.069845:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205151.069846:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205151.069847:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205151.069847:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:3.0:1788205151.069848:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205151.069849:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205151.069850:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205151.069853:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205151.069853:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069854:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.069857:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205151.069857:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.069858:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205151.069860:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205151.900243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205151.900243:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205151.900249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205151.900254:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205151.900255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205151.900259:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205151.900261:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205151.900263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205151.900265:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205151.900266:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205151.900267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205151.900271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205151.900274:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205151.900275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205151.900276:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205151.900279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205151.900281:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.900281:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205151.900284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205151.900288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205151.900289:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.900289:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205151.900294:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205151.900295:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205151.900300:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205151.900301:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205151.900462:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205151.900466:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205151.900469:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205152.412237:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321511173, at 4321511232 00000100:00000040:3.0:1788205152.412237:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321511173, at 4321511232 00000100:00000010:3.0:1788205152.412255:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc840. 00000100:00000010:2.0:1788205152.412258:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc360. 00000100:00000001:2.0:1788205152.924252:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205152.924252:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205152.924257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205152.924262:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205152.924263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205152.924264:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205152.924267:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205152.924268:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205152.924273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205152.924274:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205152.924274:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205152.924277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205152.924282:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205152.924283:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205152.924284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205152.924288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205152.924290:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205152.924290:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205152.924292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205152.924296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205152.924298:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205152.924298:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205152.924304:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205152.924305:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205152.924309:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205152.924311:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205152.924399:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205152.924404:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205152.924407:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205153.948232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205153.948232:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205153.948240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205153.948242:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205153.948243:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205153.948244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205153.948248:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205153.948248:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205153.948251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205153.948254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205153.948258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205153.948258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205153.948258:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205153.948261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205153.948266:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205153.948267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205153.948267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205153.948271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205153.948273:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205153.948273:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205153.948279:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205153.948280:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205153.948285:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205153.948285:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205153.948291:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205153.948291:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205153.948349:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205153.948354:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205153.948359:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205154.972229:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205154.972229:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205154.972236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205154.972239:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205154.972240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205154.972241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205154.972244:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.972244:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205154.972253:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205154.972254:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205154.972257:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205154.972261:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205154.972262:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205154.972262:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205154.972268:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205154.972269:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205154.972269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205154.972276:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205154.972276:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205154.972277:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205154.972282:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205154.972282:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205154.972282:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205154.972289:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205154.972290:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205154.972290:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972292:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205154.972302:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205154.972304:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205154.972306:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972307:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205154.972309:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972311:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972313:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972317:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205154.972320:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205154.972322:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205154.972324:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972326:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205154.972327:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972328:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205154.972330:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205154.972389:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205154.972394:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205154.972397:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205155.996235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205155.996238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205155.996244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205155.996247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205155.996250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205155.996251:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205155.996253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205155.996257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205155.996258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205155.996259:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205155.996261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205155.996264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205155.996267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205155.996273:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205155.996276:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205155.996279:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205155.996283:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205155.996333:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205155.996338:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205155.996341:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205155.996370:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205155.996374:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205155.996375:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205155.996378:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205155.996380:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205155.996382:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205155.996384:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205155.996386:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205155.996388:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205156.188247:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205156.188263:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205156.188266:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205156.188271:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205156.188277:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205156.188286:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26842 00000400:00000200:2.0:1788205156.188299:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205156.188308:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205156.188312:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205156.188316:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188319:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205156.188322:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188325:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205156.188330:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205156.188333:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205156.188336:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205156.188340:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205156.188341:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205156.188343:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205156.188346:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188347:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205156.188349:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188351:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205156.188353:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205156.188356:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188358:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205156.188360:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205156.188363:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205156.188365:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.188367:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205156.188369:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205156.188379:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788205156.188382:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205156.188386:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205156.188388:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.188391:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205156.188394:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205156.188398:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205156.188407:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788205156.188415:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205156.188426:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205156.188440:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205156.188445:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205156.188451:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205156.188472:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205156.188474:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205156.188481:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205156.188482:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205156.188483:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205156.188485:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205156.188489:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26842 00000400:00000200:2.0:1788205156.188492:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205156.188494:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205156.188496:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205156.188499:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188500:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205156.188503:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188504:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205156.188506:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205156.188506:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788205156.188508:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788205156.188509:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1788205156.188510:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788205156.188514:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205156.188516:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788205156.188517:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205156.188520:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1788205156.188523:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188527:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205156.188529:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205156.188532:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188538:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205156.188544:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:3.0:1788205156.188544:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205156.188548:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205156.188551:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788205156.188553:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205156.188557:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.188557:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205156.188562:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205156.188563:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205156.188567:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543517824:192.168.201.118@tcp:250:kworker.0 02000000:00000001:2.0:1788205156.188570:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205156.188571:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788205156.188574:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.188576:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788205156.188577:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205156.188581:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788205156.188582:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205156.188584:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788205156.188586:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.188588:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205156.188590:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000010:2.0:1788205156.188591:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 02000000:00000001:3.0:1788205156.188594:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205156.188595:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205156.188598:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205156.188600:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.188601:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205156.188603:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205156.188605:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.188605:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205156.188612:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 02000000:00000010:2.0:1788205156.188613:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:2.0:1788205156.188618:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205156.188620:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00100000:2.0:1788205156.188625:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788205156.188628:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543517824, portal 25 00000100:00000001:3.0:1788205156.188633:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205156.188635:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 00000100:00000001:2.0:1788205156.188640:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788205156.188644:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205156.188646:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205156.188655:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.188655:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205156.188663:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205156.188667:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000001:2.0:1788205156.188671:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205156.188673:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543517824, offset 0 00000100:00080000:2.0:1788205156.188675:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205156.188679:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788205156.188680:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205156.188682:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205156.188686:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205156.188686:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788205156.188690:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26842 00000400:00000200:2.0:1788205156.188694:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205156.188695:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205156.188697:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205156.188700:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205156.188702:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788205156.188703:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205156.188706:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188708:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205156.188710:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205156.188711:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788205156.188713:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1788205156.188715:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000040:2.0:1788205156.188716:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788205156.188719:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788205156.188723:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205156.188725:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1788205156.188727:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188729:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205156.188730:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188731:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205156.188734:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788205156.188734:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00080000:2.0:1788205156.188738:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188740:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205156.188742:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205156.188743:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1788205156.188745:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205156.188748:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.188749:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188750:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205156.188752:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205156.188755:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.188756:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471394:471394:256:4294967295] 192.168.201.118@tcp LPNI seq info [471253:471253:8:4294967295] 00000100:00000001:2.0:1788205156.188758:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205156.188763:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000400:00000200:3.0:1788205156.188765:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:2.0:1788205156.188766:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205156.188769:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205156.188770:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.188772:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205156.188773:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205156.188775:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.188779:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000010:2.0:1788205156.188780:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788205156.188786:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205156.188790:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00100000:2.0:1788205156.188793:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:3.0:1788205156.188797:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000100:00000001:2.0:1788205156.188799:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205156.188802:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205156.188804:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788205156.188809:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788205156.188814:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205156.188816:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00080000:2.0:1788205156.188820:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.1:1788205156.188826:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:2.0:1788205156.188827:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205156.188830:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:3.1:1788205156.188833:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000100:00080000:2.0:1788205156.188834:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205156.188841:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:3.0:1788205156.188842:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788205156.188847:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26842 00000800:00000010:3.0:1788205156.188848:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:2.0:1788205156.188853:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205156.188858:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205156.188858:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1788205156.188864:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205156.188869:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1788205156.188870:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188874:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205156.188875:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205156.188878:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205156.188882:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:2.0:1788205156.188883:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205156.188886:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205156.188887:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788205156.188888:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205156.188890:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788205156.188892:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205156.188893:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788205156.188895:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188897:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205156.188897:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1788205156.188901:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188903:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205156.188905:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00000200:3.0:1788205156.188907:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205156.188909:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205156.188914:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205156.188918:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205156.188922:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205156.188925:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.188926:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788205156.188927:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205156.188933:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000040:3.0:1788205156.188933:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788205156.188938:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.188943:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.188949:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.188950:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:2.0:1788205156.188957:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000100:00000001:3.0:1788205156.188958:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.188965:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:2.0:1788205156.188966:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205156.188972:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205156.188972:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321516293 02000000:00000001:2.0:1788205156.188974:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.188979:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205156.188980:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:3.0:1788205156.188981:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc840. 02000000:00000001:2.0:1788205156.188983:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205156.188990:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788205156.188994:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.188997:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00100000:2.0:1788205156.188999:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205156.189006:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321516293 00000100:00000001:2.0:1788205156.189008:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000010:3.0:1788205156.189011:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fce40. 00000100:00000001:2.0:1788205156.189012:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189016:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000040:2.0:1788205156.189021:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189021:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205156.189026:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189031:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788205156.189036:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205156.189037:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205156/real 1788205156] req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205156.189042:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205156.189049:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205156.189052:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205156.189055:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:3.0:1788205156.189057:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788205156.189059:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:3.0:1788205156.189061:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1788205156.189065:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26842 00000400:00000010:3.0:1788205156.189066:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205156.189071:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000200:2.0:1788205156.189074:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205156.189075:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205156.189127:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205156.189133:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205156.189140:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000200:3.0:1788205156.189144:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205156.189145:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205156.189152:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.189156:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205156.189156:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205156.189160:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205156.189161:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205156.189165:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205156.189166:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205156.189169:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788205156.189170:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205156.189172:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788205156.189176:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:3.0:1788205156.189177:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205156.189181:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:3.0:1788205156.189185:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788205156.189186:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205156.189191:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205156.189192:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788205156.189201:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205156.189207:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205156.189211:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1788205156.189212:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205156.189216:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205156.189217:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205156.189221:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1788205156.189223:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:3.0:1788205156.189223:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189226:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205156.189229:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788205156.189232:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00080000:3.0:1788205156.189233:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788205156.189239:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205156.189240:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 02000000:00000001:2.0:1788205156.189244:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205156.189244:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1788205156.189249:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1788205156.189249:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205156.189254:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205156.189260:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00100000:3.0:1788205156.189261:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543517824:192.168.201.118@tcp:250:kworker.0 00000008:00100000:2.0:1788205156.189264:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1788205156.189266:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788205156.189268:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189269:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788205156.189273:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205156.189274:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543517824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788205156.189281:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000100:00000001:3.0:1788205156.189281:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205156.189285:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:2.0:1788205156.189286:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000010:3.0:1788205156.189288:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:2.0:1788205156.189293:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205156.189293:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205156.189295:0:256132:0:(genops.c:1239:class_import_put()) Process entered 02000000:00000001:2.0:1788205156.189297:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205156.189298:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000100:00000001:2.0:1788205156.189302:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1788205156.189302:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000001:2.0:1788205156.189306:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:3.0:1788205156.189307:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067ec00. 02000000:00000001:2.0:1788205156.189309:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.189313:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:2.0:1788205156.189315:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 02000000:00000001:3.0:1788205156.189316:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205156.189319:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:2.0:1788205156.189322:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189322:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205156.189325:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189327:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205156.189332:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205156.189336:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205156.189339:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189343:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205156.189346:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205156.189349:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205156.189349:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205156.189352:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205156.189358:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205156.189359:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205156.189375:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189380:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205156.189382:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.189384:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205156.189386:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205156.189389:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205156.189390:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26852) 00000100:00100000:3.0:1788205156.189395:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205156.189400:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205156.189402:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205156.189404:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.189406:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205156.189410:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205156.189412:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.189416:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205156.189419:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205156.189423:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205156.189426:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543517952, portal 10 00000100:00000001:3.0:1788205156.189428:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205156.189429:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000040:3.0:1788205156.189437:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189443:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205156.189445:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205156.189448:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205156.189450:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543517952, offset 0 00000400:00000200:3.0:1788205156.189454:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205156.189458:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205156.189462:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205156.189466:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205156.189468:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205156.189471:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205156.189474:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205156.189476:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471395:471395:256:4294967295] 192.168.201.118@tcp LPNI seq info [471254:471254:8:4294967295] 00000400:00000200:3.0:1788205156.189479:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205156.189484:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205156.189487:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205156.189490:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205156.189492:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205156.189494:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205156.189497:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205156.189499:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205156.189502:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205156.189503:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205156.189505:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205156.189507:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205156.189510:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205156.189511:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205156.189513:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205156.189515:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189520:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205156.189522:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189526:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189527:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205156.189529:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205156.189532:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205156.189534:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205156.189535:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321515027 00000100:00000001:3.0:1788205156.189538:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321516293 00000100:00000001:3.0:1788205156.189539:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321515027 00000100:00000001:3.0:1788205156.189540:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321516293 00000100:00000001:3.0:1788205156.189542:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205156.189543:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189544:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205156.189546:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205156/real 1788205156] req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205156.189550:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205156.189551:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205156.189552:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205156.189553:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205156.189554:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205156.189556:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205156.189557:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205156.189563:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189566:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205156.189567:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189568:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205156.189569:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205156.189571:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189574:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205156.189576:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189580:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205156.189581:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189583:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205156.189584:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189585:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205156.189587:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205156.189589:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205156.189591:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205156.189592:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205156.189594:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205156.189599:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543517952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205156.189601:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205156.189603:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189606:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205156.189607:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.189608:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205156.189609:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205156.189610:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205156.189612:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205156.189613:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205156.189615:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205156.189616:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.189617:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205156.189619:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205156.189620:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.189622:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205156.189623:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205156.189625:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205156.189627:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543518080, portal 10 00000100:00000001:3.0:1788205156.189628:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205156.189629:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000100:00000040:3.0:1788205156.189632:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189636:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205156.189638:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205156.189639:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205156.189641:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543518080, offset 0 00000400:00000200:3.0:1788205156.189643:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205156.189645:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205156.189647:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205156.189648:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205156.189650:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205156.189651:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205156.189653:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205156.189655:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471396:471396:256:4294967295] 192.168.201.118@tcp LPNI seq info [471255:471255:8:4294967295] 00000400:00000200:3.0:1788205156.189658:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205156.189661:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205156.189664:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205156.189666:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205156.189668:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205156.189669:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205156.189672:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205156.189673:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205156.189676:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205156.189677:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205156.189678:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205156.189681:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205156.189682:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205156.189683:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205156.189685:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205156.189687:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189695:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205156.189697:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189701:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189702:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205156.189703:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205156.189705:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205156.189706:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205156.189707:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321515028 00000100:00000001:3.0:1788205156.189708:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321516293 00000100:00000001:3.0:1788205156.189710:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321515028 00000100:00000001:3.0:1788205156.189711:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321516293 00000100:00000001:3.0:1788205156.189713:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205156.189714:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189715:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205156.189716:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205156/real 1788205156] req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205156.189720:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205156.189721:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205156.189722:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205156.189723:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205156.189724:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205156.189726:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205156.189727:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205156.189732:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189735:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205156.189736:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189737:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205156.189738:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205156.189740:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189744:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205156.189746:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189749:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205156.189750:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189751:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205156.189752:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189753:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205156.189754:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205156.189756:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205156.189757:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205156.189758:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205156.189760:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205156.189764:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205156.189766:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205156.189768:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189772:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205156.189773:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.189774:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205156.189775:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205156.189776:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205156.189779:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205156.189781:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205156.189782:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205156.189783:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.189785:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205156.189788:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205156.189790:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.189793:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205156.189795:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205156.189798:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205156.189801:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543518208, portal 4 00000100:00000001:3.0:1788205156.189802:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205156.189804:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:3.0:1788205156.189810:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189816:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205156.189818:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205156.189821:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205156.189823:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543518208, offset 0 00000400:00000200:3.0:1788205156.189826:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205156.189830:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205156.189833:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205156.189836:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205156.189838:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205156.189841:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205156.189843:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205156.189846:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471397:471397:256:4294967295] 192.168.201.118@tcp LPNI seq info [471256:471256:8:4294967295] 00000400:00000200:3.0:1788205156.189850:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205156.189855:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205156.189860:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205156.189863:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205156.189867:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205156.189870:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205156.189874:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205156.189876:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205156.189880:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205156.189882:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205156.189885:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205156.189888:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205156.189891:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205156.189893:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205156.189896:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205156.189899:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189907:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205156.189910:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189916:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189918:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205156.189921:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205156.189924:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205156.189927:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205156.189929:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321515037 00000100:00000001:3.0:1788205156.189931:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321516293 00000100:00000001:3.0:1788205156.189934:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321515037 00000100:00000001:3.0:1788205156.189936:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321516293 00000100:00000001:3.0:1788205156.189938:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205156.189939:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.189941:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205156.189943:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205156/real 1788205156] req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205156.189950:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205156.189951:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205156.189953:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205156.189955:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205156.189957:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205156.189959:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205156.189962:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205156.189969:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189975:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205156.189977:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189978:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205156.189980:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205156.189983:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189988:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205156.189990:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.189994:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205156.189995:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189996:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205156.189997:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.189998:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205156.189999:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205156.190001:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205156.190002:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205156.190004:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205156.190006:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205156.190015:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205156.190017:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190018:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205156.190020:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543517952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190023:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205156.190025:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205156.190026:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205156.190027:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205156.190029:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205156.190030:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205156.190031:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205156.190032:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:3.0:1788205156.190034:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205156.190035:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205156.190036:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:3.0:1788205156.190038:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205156.190039:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190040:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205156.190042:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543518080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190046:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205156.190047:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205156.190048:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205156.190049:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205156.190050:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205156.190050:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205156.190051:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205156.190052:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:3.0:1788205156.190053:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205156.190054:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205156.190055:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:3.0:1788205156.190057:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205156.190058:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190059:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205156.190061:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543518208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190064:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205156.190065:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205156.190066:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205156.190067:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205156.190067:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205156.190068:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205156.190069:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205156.190070:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:3.0:1788205156.190071:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205156.190071:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205156.190072:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:3.0:1788205156.190073:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205156.190074:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190075:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190077:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205156.190078:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.190079:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205156.190105:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205156.190106:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205156.190109:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190112:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205156.190113:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.190115:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205156.190116:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205156.190117:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205156.190118:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205156.190121:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205156.190122:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205156.190123:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205156.190125:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205156.190127:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205156.190128:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205156.190130:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205156.190132:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205156.190134:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205156.190136:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543518336, portal 4 00000100:00000001:3.0:1788205156.190137:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205156.190138:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:3.0:1788205156.190142:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190146:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205156.190148:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205156.190149:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205156.190150:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543518336, offset 0 00000400:00000200:3.0:1788205156.190152:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205156.190155:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205156.190157:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205156.190158:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205156.190160:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205156.190161:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205156.190163:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205156.190165:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471398:471398:256:4294967295] 192.168.201.118@tcp LPNI seq info [471257:471257:8:4294967295] 00000400:00000200:3.0:1788205156.190167:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205156.190171:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205156.190174:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205156.190176:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205156.190178:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205156.190180:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205156.190182:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205156.190184:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205156.190186:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205156.190188:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205156.190190:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205156.190192:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205156.190194:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205156.190195:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205156.190196:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205156.190198:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190203:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205156.190205:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205181 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190209:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.190210:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205156.190212:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205156.190213:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205156.190215:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205156.190216:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321515061 00000100:00000001:3.0:1788205156.190217:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321516293 00000100:00000001:3.0:1788205156.190219:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321515061 00000100:00000001:3.0:1788205156.190220:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321516293 00000100:00000001:3.0:1788205156.190222:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205156.190223:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.190224:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205156.190226:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205156/real 1788205156] req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205156.190230:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205156.190230:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205156.190232:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205156.190233:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205156.190234:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205156.190235:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205156.190237:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205156.190242:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190246:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205156.190247:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190247:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205156.190248:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205156.190250:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190253:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205156.190255:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190259:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205156.190260:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190261:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205156.190261:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190262:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205156.190263:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205156.190265:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205156.190267:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205156.190268:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205156.190270:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205156.190275:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205156.190277:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190278:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205156.190280:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77b80 x1875055543518336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205181 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205156.190284:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205156.190285:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205156.190286:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205156.190288:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205156.190288:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205156.190289:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205156.190290:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205156.190291:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e800. 02000000:00000001:3.0:1788205156.190293:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205156.190293:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205156.190294:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77b80. 00000100:00000001:3.0:1788205156.190296:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205156.190296:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190298:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205156.190300:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205156.190300:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205156.190302:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205156.190303:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205157.020243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205157.020243:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205157.020246:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205157.020250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205157.020253:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205157.020255:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205157.020257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205157.020258:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205157.020260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205157.020261:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205157.020266:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205157.020267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205157.020268:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205157.020271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205157.020274:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205157.020275:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205157.020280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205157.020285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205157.020285:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205157.020286:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205157.020300:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205157.020303:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205157.020304:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205157.020311:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205157.020312:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205157.020325:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205157.020378:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205157.020381:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205157.020384:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205157.532160:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321516293, at 4321516352 00000100:00000010:2.0:1788205157.532177:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc840. 00000100:00000040:3.0:1788205157.532207:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321516293, at 4321516352 00000100:00000010:3.0:1788205157.532221:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fce40. 00000100:00000001:2.0:1788205158.044238:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205158.044238:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205158.044245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205158.044247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205158.044248:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205158.044251:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205158.044252:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205158.044253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205158.044256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205158.044260:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205158.044261:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205158.044262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205158.044266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205158.044266:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205158.044266:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205158.044271:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205158.044272:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205158.044274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205158.044276:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205158.044278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205158.044278:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205158.044280:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205158.044283:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205158.044286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205158.044288:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205158.044289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205158.044370:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205158.044375:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205158.044379:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205159.068235:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205159.068240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205159.068245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.068246:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205159.068249:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205159.068254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205159.068255:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205159.068257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.068263:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205159.068264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205159.068267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205159.068271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205159.068272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205159.068276:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.068281:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205159.068284:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205159.068289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205159.068334:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205159.068339:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205159.068342:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205159.068619:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205159.068623:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205159.068624:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205159.068626:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205159.068627:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205159.068629:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205159.068631:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205159.068632:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205159.068634:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205160.092219:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205160.092225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205160.092228:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205160.092233:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205160.092233:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205160.092233:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205160.092239:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205160.092244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205160.092245:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205160.092245:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205160.092249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205160.092253:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205160.092254:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00100000:2.0:1788205160.092258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205160.092259:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205160.092260:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205160.092268:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205160.092269:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205160.092270:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205160.092275:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205160.092276:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205160.092276:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205160.092285:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205160.092286:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205160.092289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205160.092292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205160.092293:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205160.092298:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205160.092301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205160.092306:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205160.092311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205160.092313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205160.092374:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205160.092378:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205160.092382:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205161.116231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205161.116232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205161.116242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205161.116245:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205161.116250:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205161.116253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205161.116257:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205161.116258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205161.116268:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205161.116269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205161.116275:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205161.116276:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205161.116282:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205161.116283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205161.116288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205161.116288:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205161.116299:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.116319:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205161.116324:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205161.116326:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205161.116330:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205161.116335:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205161.116338:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.116341:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.116343:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.116346:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205161.116392:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205161.116396:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205161.116399:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205161.308268:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205161.308275:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205161.308277:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205161.308280:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205161.308284:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205161.308291:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26847 00000400:00000200:1.0:1788205161.308300:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205161.308307:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205161.308311:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205161.308314:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308318:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.308320:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308323:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205161.308327:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205161.308329:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205161.308331:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205161.308335:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205161.308336:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205161.308338:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205161.308340:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308342:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.308344:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308345:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205161.308348:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205161.308352:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308354:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205161.308356:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205161.308360:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205161.308362:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308365:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205161.308367:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205161.308376:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205161.308380:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205161.308384:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205161.308386:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308389:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205161.308391:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205161.308394:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205161.308400:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9800. 00000100:00000001:1.0:1788205161.308406:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205161.308412:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205161.308420:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205161.308448:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205161.308453:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205161.308485:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205161.308487:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205161.308492:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205161.308493:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205161.308494:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205161.308496:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205161.308498:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26847 00000400:00000200:1.0:1788205161.308506:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205161.308509:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205161.308511:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205161.308513:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308515:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.308516:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205161.308517:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205161.308518:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205161.308522:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:3.0:1788205161.308523:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1788205161.308524:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205161.308526:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:3.0:1788205161.308527:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1788205161.308529:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205161.308531:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205161.308534:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205161.308535:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205161.308538:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308542:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.308546:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205161.308549:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.308552:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308553:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205161.308557:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 02000000:00000001:3.0:1788205161.308557:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1788205161.308561:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788205161.308561:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205161.308566:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308567:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205161.308570:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788205161.308573:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543518464:192.168.201.118@tcp:250:kworker.0 02000000:00000001:1.0:1788205161.308576:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205161.308577:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205161.308581:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.308582:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205161.308584:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.308586:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205161.308590:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205161.308591:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205161.308596:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205161.308598:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205161.308603:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205161.308608:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543518464, portal 25 00000100:00000001:1.0:1788205161.308611:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205161.308611:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205161.308614:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000008:00100000:1.0:1788205161.308617:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000040:3.0:1788205161.308619:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1788205161.308623:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308628:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.308635:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:1.0:1788205161.308636:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788205161.308641:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000040:3.0:1788205161.308641:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000001:1.0:1788205161.308645:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205161.308646:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308647:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1788205161.308647:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 02000000:00000001:1.0:1788205161.308650:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205161.308653:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205161.308653:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543518464, offset 0 02000000:00000010:1.0:1788205161.308657:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9c00. 00000100:00000001:1.0:1788205161.308661:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205161.308664:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00100000:1.0:1788205161.308669:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205161.308676:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788205161.308677:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205161.308679:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205161.308684:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205161.308686:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205161.308694:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1788205161.308696:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205161.308699:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00080000:1.0:1788205161.308700:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205161.308704:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1788205161.308705:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:3.0:1788205161.308708:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205161.308712:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471399:471399:256:4294967295] 192.168.201.118@tcp LPNI seq info [471258:471258:8:4294967295] 00000100:00000001:1.0:1788205161.308714:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788205161.308717:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:1.0:1788205161.308718:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205161.308721:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205161.308725:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26847 00000400:00000200:1.0:1788205161.308733:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205161.308733:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205161.308740:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205161.308742:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205161.308744:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000010:3.0:1788205161.308746:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000100:00000040:1.0:1788205161.308747:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308750:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.308752:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308754:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.0:1788205161.308755:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:1.0:1788205161.308760:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000200:3.0:1788205161.308760:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000020:00000040:1.0:1788205161.308763:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205161.308765:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000200:3.1:1788205161.308767:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:1.0:1788205161.308768:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000800:00000200:3.1:1788205161.308772:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000020:00000001:1.0:1788205161.308773:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205161.308778:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000800:00000100:3.0:1788205161.308781:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000040:1.0:1788205161.308784:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000010:3.0:1788205161.308785:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000100:00000001:1.0:1788205161.308791:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205161.308791:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1788205161.308794:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.308797:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205161.308799:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000020:00000040:1.0:1788205161.308800:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205161.308805:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:3.0:1788205161.308805:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205161.308809:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:1.0:1788205161.308810:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205161.308813:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205161.308815:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1788205161.308819:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:3.0:1788205161.308820:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205161.308823:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308827:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205161.308830:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205161.308834:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308836:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.308836:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.308842:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1788205161.308845:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205161.308847:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205161.308851:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000100:00000001:3.0:1788205161.308851:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1788205161.308855:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205161.308857:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.308857:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.308858:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205161.308861:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205161.308862:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205161.308863:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000010:1.0:1788205161.308867:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb000. 00000100:00000001:3.0:1788205161.308867:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321521413 00000100:00000010:3.0:1788205161.308874:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fce40. 00000100:00000001:1.0:1788205161.308875:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.308879:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00100000:1.0:1788205161.308881:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205161.308886:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321521413 00000100:00000001:1.0:1788205161.308887:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205161.308890:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205161.308892:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc0c0. 00000100:00000040:1.0:1788205161.308895:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.308896:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205161.308899:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205161.308901:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.308903:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205161.308906:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205161/real 1788205161] req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205161.308921:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205161.308926:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205161.308936:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:3.0:1788205161.308937:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788205161.308939:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:3.0:1788205161.308940:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1788205161.308942:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:3.0:1788205161.308943:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205161.308947:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000100:00000001:1.0:1788205161.308949:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:3.0:1788205161.308950:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205161.308954:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:1.0:1788205161.308956:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26847 00000100:00000200:3.0:1788205161.308959:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205161.308971:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.308977:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205161.308978:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.308980:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.308982:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205161.308985:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205161.308987:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205161.309006:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205161.309011:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205161.309012:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205161.309015:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.309018:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.309021:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205161.309022:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205161.309023:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205161.309027:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205161.309029:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205161.309032:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205161.309034:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00000001:3.0:1788205161.309039:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205161.309042:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309043:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.309044:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309045:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205161.309047:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205161.309049:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:1.0:1788205161.309051:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205161.309055:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00080000:3.0:1788205161.309055:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205161.309060:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205161.309061:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.309064:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.309068:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.309069:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788205161.309069:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205161.309074:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205161.309078:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205161.309082:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205161.309086:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205161.309089:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788205161.309126:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543518464:192.168.201.118@tcp:250:kworker.0 02000000:00000001:1.0:1788205161.309129:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309133:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309137:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.309141:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543518464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309150:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1788205161.309151:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205161.309153:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000008:00100000:1.0:1788205161.309155:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 02000000:00000010:3.0:1788205161.309158:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 00000008:00000001:1.0:1788205161.309159:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.309163:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.309164:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205161.309167:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205161.309169:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205161.309171:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205161.309173:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:3.0:1788205161.309177:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205161.309179:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205161.309181:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000100:00000010:3.0:1788205161.309182:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205161.309186:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788205161.309190:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1788205161.309191:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205161.309195:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309195:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205161.309201:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205161.309201:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788205161.309204:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205161.309207:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205161.309211:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8400. 00000100:00000001:1.0:1788205161.309214:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309218:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1788205161.309220:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205161.309222:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309228:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205161.309231:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205161.309233:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205161.309238:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309246:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.309247:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.309250:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788205161.309251:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788205161.309254:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205161.309260:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205161.309262:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205161.309267:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205161.309274:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518592:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205161.309278:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309280:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1788205161.309283:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788205161.309285:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205161.309288:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205161.309289:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:3.0:1788205161.309293:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788205161.309294:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205161.309298:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:3.0:1788205161.309300:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:1.0:1788205161.309301:0:409030:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1788205161.309306:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205161.309307:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26847 00000400:00000200:3.0:1788205161.309313:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205161.309316:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000200:1.0:1788205161.309321:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205161.309322:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000400:00000200:1.0:1788205161.309326:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000200:3.0:1788205161.309328:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543518592, portal 10 00000100:00080000:1.0:1788205161.309330:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205161.309333:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1788205161.309337:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205161.309337:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000001:1.0:1788205161.309341:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.309351:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.309354:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788205161.309355:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205161.309356:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205161.309359:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205161.309362:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205161.309366:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:3.0:1788205161.309369:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205161.309372:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205161.309374:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205161.309375:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543518592, offset 0 00000400:00000200:3.0:1788205161.309378:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205161.309381:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205161.309385:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:1.0:1788205161.309388:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000200:3.0:1788205161.309389:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205161.309393:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000020:00000001:1.0:1788205161.309394:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000200:3.0:1788205161.309396:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1788205161.309402:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205161.309403:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.309408:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205161.309413:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205161.309415:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205161.309417:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000200:3.0:1788205161.309417:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471400:471400:256:4294967295] 192.168.201.118@tcp LPNI seq info [471259:471259:8:4294967295] 00000100:00080000:1.0:1788205161.309425:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:3.0:1788205161.309429:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205161.309430:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205161.309434:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205161.309438:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205161.309442:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:1.0:1788205161.309443:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.309447:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205161.309451:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000800:00000200:3.0:1788205161.309452:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000008:00000001:1.0:1788205161.309458:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205161.309460:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000100:00000001:1.0:1788205161.309461:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205161.309466:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000800:00000200:3.0:1788205161.309466:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:1.0:1788205161.309470:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000800:00000200:3.0:1788205161.309471:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 02000000:00000001:1.0:1788205161.309473:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205161.309475:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205161.309478:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:3.1:1788205161.309478:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:1.0:1788205161.309481:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1788205161.309482:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 02000000:00000001:1.0:1788205161.309484:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788205161.309488:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000010:1.0:1788205161.309490:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8800. 00000800:00000010:3.0:1788205161.309490:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000100:00000001:1.0:1788205161.309496:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205161.309496:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205161.309503:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00100000:1.0:1788205161.309506:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205161.309507:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205161.309510:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205161.309514:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1788205161.309518:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000200:3.0:1788205161.309521:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205161.309522:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205161.309529:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309535:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788205161.309539:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205161.309542:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205161.309544:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26857) 00000100:00000001:3.0:1788205161.309551:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309553:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205161.309555:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205161.309557:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205161.309560:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205161.309562:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321520147 00000100:00000001:3.0:1788205161.309565:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321521413 00000100:00000001:3.0:1788205161.309568:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321520147 00000100:00000001:3.0:1788205161.309570:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321521413 00000100:00000001:3.0:1788205161.309572:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205161.309574:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309575:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205161.309579:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205161/real 1788205161] req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205161.309586:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205161.309588:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205161.309590:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205161.309592:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205161.309593:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205161.309595:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205161.309598:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205161.309607:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309613:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205161.309615:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309616:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.309617:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205161.309621:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309627:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205161.309630:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309636:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205161.309639:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309640:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.309642:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309643:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205161.309645:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205161.309648:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205161.309651:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205161.309653:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205161.309657:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205161.309665:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518592:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205161.309669:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205161.309671:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309675:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.309675:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.309677:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205161.309678:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205161.309679:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205161.309681:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518720:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205161.309683:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205161.309684:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205161.309685:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.309687:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205161.309689:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205161.309690:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205161.309692:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205161.309694:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205161.309696:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205161.309699:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543518720, portal 10 00000100:00000001:3.0:1788205161.309700:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205161.309701:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205161.309705:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309708:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205161.309710:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205161.309711:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205161.309713:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543518720, offset 0 00000400:00000200:3.0:1788205161.309715:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205161.309717:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205161.309719:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205161.309721:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205161.309723:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205161.309725:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205161.309726:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205161.309728:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471401:471401:256:4294967295] 192.168.201.118@tcp LPNI seq info [471260:471260:8:4294967295] 00000400:00000200:3.0:1788205161.309730:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205161.309734:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205161.309736:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205161.309738:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205161.309741:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205161.309742:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205161.309745:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205161.309747:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205161.309749:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205161.309750:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205161.309752:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205161.309754:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205161.309756:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205161.309757:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205161.309758:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205161.309760:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309765:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.309766:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309770:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309771:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205161.309773:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205161.309775:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205161.309776:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205161.309778:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321520148 00000100:00000001:3.0:1788205161.309779:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321521413 00000100:00000001:3.0:1788205161.309781:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321520148 00000100:00000001:3.0:1788205161.309782:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321521413 00000100:00000001:3.0:1788205161.309784:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205161.309785:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309786:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205161.309788:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205161/real 1788205161] req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205161.309792:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205161.309793:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205161.309794:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205161.309795:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205161.309796:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205161.309797:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205161.309798:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205161.309803:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309807:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205161.309808:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309809:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.309809:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205161.309811:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309814:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205161.309816:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309820:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205161.309821:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309822:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.309822:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309824:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205161.309826:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205161.309827:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205161.309829:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205161.309831:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205161.309834:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205161.309838:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518720:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205161.309840:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309841:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.309842:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543518592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309846:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205161.309846:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205161.309848:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205161.309849:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205161.309850:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205161.309850:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205161.309852:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205161.309853:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:3.0:1788205161.309854:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205161.309855:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205161.309856:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205161.309860:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205161.309861:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309867:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.309869:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543518720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309872:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205161.309873:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205161.309874:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205161.309876:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205161.309877:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205161.309877:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205161.309879:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205161.309879:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:3.0:1788205161.309881:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205161.309881:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205161.309882:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205161.309883:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205161.309884:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309885:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.309888:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205161.309889:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.309890:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205161.309891:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205161.309892:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205161.309893:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309898:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.309899:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.309900:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205161.309901:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205161.309902:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205161.309903:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518848:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205161.309905:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205161.309906:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205161.309907:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.309908:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205161.309910:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205161.309911:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205161.309913:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205161.309914:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205161.309916:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205161.309918:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543518848, portal 4 00000100:00000001:3.0:1788205161.309919:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205161.309920:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205161.309923:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.309928:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205161.309929:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205161.309930:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205161.309932:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543518848, offset 0 00000400:00000200:3.0:1788205161.309934:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205161.309936:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205161.309938:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205161.309939:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205161.309940:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205161.309942:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205161.309943:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205161.309945:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471402:471402:256:4294967295] 192.168.201.118@tcp LPNI seq info [471261:471261:8:4294967295] 00000400:00000200:3.0:1788205161.309948:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205161.309951:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205161.309954:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205161.309956:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205161.309959:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205161.309961:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205161.309965:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205161.309967:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205161.309970:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205161.309972:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205161.309975:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205161.309978:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205161.309980:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205161.309982:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205161.309984:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205161.309987:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310005:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.310009:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310016:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.310017:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205161.310020:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205161.310023:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205161.310026:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205161.310028:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321520157 00000100:00000001:3.0:1788205161.310031:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321521413 00000100:00000001:3.0:1788205161.310032:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321520157 00000100:00000001:3.0:1788205161.310034:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321521413 00000100:00000001:3.0:1788205161.310035:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205161.310036:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.310037:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205161.310040:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205161/real 1788205161] req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205161.310045:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205161.310046:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205161.310047:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205161.310049:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205161.310050:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205161.310051:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205161.310053:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205161.310058:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310062:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205161.310063:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310064:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.310064:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205161.310067:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310071:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205161.310073:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310076:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205161.310077:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310078:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.310079:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310080:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205161.310081:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205161.310082:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205161.310084:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205161.310085:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205161.310087:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205161.310109:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518848:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205161.310112:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205161.310115:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310121:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205161.310122:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.310124:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205161.310126:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205161.310127:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205161.310130:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518976:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205161.310132:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205161.310134:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205161.310136:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205161.310137:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205161.310140:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205161.310142:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205161.310145:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205161.310147:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205161.310150:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205161.310153:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543518976, portal 4 00000100:00000001:3.0:1788205161.310155:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205161.310156:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205161.310162:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310167:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205161.310168:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205161.310170:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205161.310171:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543518976, offset 0 00000400:00000200:3.0:1788205161.310174:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205161.310177:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205161.310179:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205161.310182:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205161.310185:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205161.310187:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205161.310189:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205161.310192:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471403:471403:256:4294967295] 192.168.201.118@tcp LPNI seq info [471262:471262:8:4294967295] 00000400:00000200:3.0:1788205161.310195:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205161.310200:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205161.310205:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205161.310208:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205161.310212:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205161.310214:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205161.310218:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205161.310220:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205161.310224:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205161.310227:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205161.310230:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205161.310233:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205161.310235:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205161.310237:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205161.310239:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205161.310241:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310246:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.310248:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205186 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310252:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.310253:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205161.310255:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205161.310257:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205161.310259:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205161.310260:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321520181 00000100:00000001:3.0:1788205161.310261:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321521413 00000100:00000001:3.0:1788205161.310263:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321520181 00000100:00000001:3.0:1788205161.310264:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321521413 00000100:00000001:3.0:1788205161.310266:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205161.310267:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.310268:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205161.310270:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205161/real 1788205161] req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205161.310274:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205161.310275:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205161.310277:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205161.310279:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205161.310280:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205161.310282:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205161.310285:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205161.310292:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310298:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205161.310300:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310302:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.310303:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205161.310306:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310313:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205161.310316:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310323:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205161.310325:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310326:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205161.310327:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310329:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205161.310330:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205161.310333:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205161.310336:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205161.310338:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205161.310342:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205161.310349:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543518976:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205161.310352:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310354:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.310358:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543518848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310364:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205161.310366:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205161.310368:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205161.310370:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205161.310371:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205161.310372:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205161.310373:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205161.310374:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:3.0:1788205161.310376:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205161.310377:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205161.310378:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205161.310382:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205161.310383:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310384:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205161.310386:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543518976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205186 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205161.310390:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205161.310391:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205161.310391:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205161.310393:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205161.310393:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205161.310394:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205161.310395:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205161.310395:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:3.0:1788205161.310396:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205161.310397:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205161.310398:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205161.310399:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205161.310399:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310400:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205161.310403:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205161.310404:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205161.310405:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205161.310406:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205162.140206:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205162.140210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205162.140215:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205162.140215:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205162.140216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205162.140221:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205162.140226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205162.140226:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205162.140228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205162.140233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205162.140234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205162.140234:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205162.140240:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205162.140242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205162.140244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205162.140245:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205162.140246:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205162.140248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205162.140251:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205162.140252:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205162.140256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205162.140259:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205162.140261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205162.140262:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205162.140263:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205162.140272:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205162.140427:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205162.140433:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205162.140438:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205162.652261:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321521413, at 4321521472 00000100:00000010:2.0:1788205162.652279:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fce40. 00000100:00000040:3.0:1788205162.652279:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321521413, at 4321521472 00000100:00000010:3.0:1788205162.652296:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc0c0. 00000100:00000001:2.0:1788205163.164228:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205163.164229:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205163.164240:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205163.164241:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205163.164243:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205163.164246:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205163.164250:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205163.164256:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788205163.164257:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205163.164259:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205163.164264:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205163.164265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205163.164267:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205163.164270:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205163.164272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205163.164274:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205163.164278:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205163.164281:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205163.164282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205163.164287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205163.164289:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205163.164295:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205163.164296:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205163.164301:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205163.164311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205163.164313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205163.164386:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205163.164391:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205163.164395:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205164.188230:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205164.188238:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205164.188241:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205164.188246:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205164.188250:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205164.188254:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.188256:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205164.188259:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205164.188262:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205164.188364:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205164.188368:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205164.188369:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205164.188373:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205164.188375:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205164.188377:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205164.188379:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205164.188381:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205164.188383:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205164.188483:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205164.188486:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205164.188491:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205164.188492:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205164.188496:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205164.188497:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205164.188501:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205164.188502:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205164.188563:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205164.188567:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205164.188571:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205165.212196:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205165.212196:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205165.212203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205165.212204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205165.212204:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205165.212207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205165.212209:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205165.212212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205165.212215:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205165.212218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205165.212218:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205165.212219:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205165.212223:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205165.212224:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205165.212227:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205165.212227:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205165.212231:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205165.212233:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205165.212236:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212239:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205165.212243:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212245:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212248:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205165.212254:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205165.212255:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205165.212257:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212259:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205165.212260:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212261:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212263:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205165.212268:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205165.212269:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205165.212270:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212272:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205165.212273:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212275:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205165.212276:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205165.212330:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205165.212333:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205165.212335:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205165.212337:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205165.212406:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205165.212409:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205165.212412:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205166.236208:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205166.236216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205166.236218:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205166.236223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.236223:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205166.236232:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205166.236233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205166.236237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.236238:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205166.236242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205166.236243:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205166.236245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.236248:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205166.236251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205166.236252:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205166.236255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.236257:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205166.236351:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205166.236355:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205166.236359:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205166.236771:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205166.236778:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205166.236780:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205166.236785:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205166.236788:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205166.236792:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205166.236794:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205166.236797:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205166.236799:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205166.428280:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205166.428294:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205166.428297:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205166.428301:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205166.428306:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205166.428315:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26852 00000400:00000200:0.0:1788205166.428384:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205166.428393:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205166.428397:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205166.428402:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428405:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.428408:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428411:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205166.428416:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205166.428418:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205166.428421:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205166.428425:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205166.428427:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205166.428430:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205166.428433:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428435:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.428437:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428439:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205166.428443:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205166.428446:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428449:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205166.428452:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205166.428457:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205166.428460:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.428463:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205166.428466:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205166.428477:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da680. 00000020:00000040:0.0:1788205166.428481:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205166.428486:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205166.428488:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.428492:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205166.428496:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205166.428501:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205166.428511:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788205166.428520:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205166.428530:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205166.428542:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205166.428547:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205166.428552:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000633fbd17] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205166.428581:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205166.428585:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205166.428588:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205166.428590:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205166.428591:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205166.428595:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205166.428598:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26852 00000400:00000200:0.0:1788205166.428602:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205166.428605:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205166.428608:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205166.428612:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428614:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.428616:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428619:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205166.428622:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205166.428623:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205166.428625:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205166.428628:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205166.428631:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205166.428631:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788205166.428636:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205166.428638:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1788205166.428641:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205166.428643:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205166.428646:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.428651:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205166.428653:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205166.428654:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205166.428661:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788205166.428668:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205166.428672:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205166.428673:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.428675:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205166.428677:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205166.428680:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205166.428685:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205166.428685:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205166.428689:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205166.428690:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205166.428695:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543519104:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788205166.428696:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205166.428701:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1788205166.428703:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205166.428708:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.428708:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788205166.428712:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.428712:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.428716:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1788205166.428719:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dbb80. 02000000:00000010:3.0:1788205166.428723:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 00000020:00000040:0.0:1788205166.428724:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205166.428729:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205166.428730:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205166.428733:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.428736:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:3.0:1788205166.428737:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:0.0:1788205166.428738:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205166.428742:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205166.428743:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 02000000:00000010:0.0:1788205166.428748:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788205166.428754:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205166.428754:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00100000:0.0:1788205166.428764:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:3.0:1788205166.428764:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543519104, portal 25 00000100:00000001:3.0:1788205166.428769:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205166.428771:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904119424 : -109419805432192 : ffff9c7bb75da680) 00000100:00000001:0.0:1788205166.428779:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:3.0:1788205166.428781:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205166.428783:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205166.428792:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f1d82a0] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.428810:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788205166.428814:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205166.428817:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:0.0:1788205166.428820:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:3.0:1788205166.428821:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205166.428825:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543519104, offset 0 00000100:00080000:0.0:1788205166.428827:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205166.428830:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205166.428832:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:3.0:1788205166.428836:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428837:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205166.428844:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26852 00000400:00000200:3.0:1788205166.428847:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205166.428851:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205166.428858:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205166.428858:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00080000:0.0:1788205166.428864:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:3.0:1788205166.428867:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1788205166.428870:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428874:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205166.428875:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000040:0.0:1788205166.428879:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205166.428882:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1788205166.428884:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205166.428887:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000001:0.0:1788205166.428889:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205166.428892:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000400:00000200:3.0:1788205166.428895:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471404:471404:256:4294967295] 192.168.201.118@tcp LPNI seq info [471263:471263:8:4294967295] 00000020:00000001:0.0:1788205166.428896:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205166.428899:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205166.428913:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788205166.428915:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788205166.428920:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428922:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.428924:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428926:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205166.428928:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788205166.428931:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205166.428933:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205166.428935:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205166.428938:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205166.428940:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.428942:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205166.428944:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205166.428947:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205166.428950:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205166.428951:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205166.428957:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9880. 00000020:00000040:0.0:1788205166.428960:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205166.428966:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:3.0:1788205166.428966:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:0.0:1788205166.428970:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.428974:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205166.428978:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.0:1788205166.428978:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:0.0:1788205166.428982:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205166.428986:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 02000000:00000010:0.0:1788205166.428988:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788205166.428994:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205166.428997:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00100000:0.0:1788205166.429003:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788205166.429003:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205166.429014:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788205166.429017:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:3.1:1788205166.429020:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000100:00000001:0.0:1788205166.429021:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205166.429030:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a6fd274] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1788205166.429031:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.429036:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205166.429044:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788205166.429045:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205166.429051:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205166.429053:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00080000:0.0:1788205166.429058:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:3.0:1788205166.429060:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000100:00000001:0.0:1788205166.429061:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:3.0:1788205166.429065:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00080000:0.0:1788205166.429066:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205166.429072:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205166.429073:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1788205166.429077:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26852 00000400:00000200:0.0:1788205166.429082:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205166.429086:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205166.429088:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205166.429119:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429122:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.429124:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429126:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205166.429128:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205166.429129:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205166.429131:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205166.429133:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788205166.429135:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788205166.429137:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788205166.429140:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429142:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.429144:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429146:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205166.429148:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788205166.429150:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429153:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205166.429155:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205166.429158:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205166.429160:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.429162:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205166.429165:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788205166.429167:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.429169:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205166.429172:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75daa00. 00000020:00000040:0.0:1788205166.429175:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205166.429177:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205166.429179:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.429181:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205166.429182:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205166.429184:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205166.429187:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788205166.429191:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205166.429196:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205166.429203:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205166.429205:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205166.429209:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000067e460ad] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205166.429216:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205166.429219:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205166.429222:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205166.429224:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205166.429226:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205166.429229:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205166.429232:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26852 00000400:00000200:0.0:1788205166.429236:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205166.429239:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205166.429241:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205166.429244:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429246:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.429248:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429250:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205166.429252:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205166.429253:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788205166.429255:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205166.429257:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205166.429259:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205166.429261:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788205166.429264:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429266:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205166.429268:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429270:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205166.429272:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788205166.429275:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205166.429277:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205166.429279:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205166.429282:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205166.429284:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.429286:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205166.429288:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205166.429291:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.429292:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205166.429295:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9c00. 00000020:00000040:0.0:1788205166.429298:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205166.429300:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205166.429301:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205166.429303:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205166.429304:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205166.429306:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205166.429310:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000100:00000001:0.0:1788205166.429313:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205166.429318:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205166.429335:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205166.429337:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205166.429341:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f503ab2e] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205166.429349:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205166.429354:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26862) 00000100:00000200:3.0:1788205166.430068:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430114:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.430118:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430131:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.430133:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205166.430141:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205166.430145:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205166.430149:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321526535 00000100:00000010:3.0:1788205166.430169:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc0c0. 00000100:00000001:3.0:1788205166.430177:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205166.430182:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321526535 00000100:00000010:3.0:1788205166.430185:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc720. 00000100:00000001:3.0:1788205166.430188:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205166.430190:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205166.430193:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.430196:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205166.430200:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205166/real 1788205166] req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205166.430209:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205166.430211:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205166.430213:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205166.430215:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205166.430217:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205166.430220:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205166.430223:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205166.430231:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430237:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205166.430239:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430241:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.430243:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205166.430246:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430253:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205166.430257:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430263:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205166.430265:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430266:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.430267:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430270:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205166.430272:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205166.430275:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205166.430278:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205166.430280:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205166.430284:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205166.430291:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543519104:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205166.430295:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430297:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.430300:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da680 x1875055543519104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430305:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205166.430308:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205166.430311:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205166.430314:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205166.430316:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205166.430317:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205166.430320:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205166.430322:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28c00. 02000000:00000001:3.0:1788205166.430326:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205166.430328:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205166.430330:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da680. 00000100:00000001:3.0:1788205166.430333:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205166.430334:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430336:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430345:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205166.430347:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.430349:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205166.430350:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205166.430352:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205166.430355:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430361:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205166.430362:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.430364:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205166.430367:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205166.430369:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205166.430372:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519232:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205166.430375:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205166.430378:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205166.430380:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.430382:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205166.430385:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205166.430386:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205166.430390:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205166.430393:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205166.430397:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205166.430400:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543519232, portal 10 00000100:00000001:3.0:1788205166.430401:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205166.430403:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904124800 : -109419805426816 : ffff9c7bb75dbb80) 00000100:00000040:3.0:1788205166.430410:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430416:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205166.430418:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205166.430421:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205166.430423:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543519232, offset 0 00000400:00000200:3.0:1788205166.430428:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205166.430432:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205166.430437:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205166.430440:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205166.430443:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205166.430446:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205166.430448:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205166.430452:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471405:471405:256:4294967295] 192.168.201.118@tcp LPNI seq info [471264:471264:8:4294967295] 00000400:00000200:3.0:1788205166.430456:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205166.430463:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205166.430468:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.430472:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205166.430476:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205166.430479:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205166.430483:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205166.430485:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205166.430489:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.430490:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205166.430493:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205166.430497:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205166.430500:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205166.430502:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205166.430504:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205166.430508:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430514:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.430517:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430522:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.430523:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205166.430527:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205166.430530:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205166.430533:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205166.430535:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321525268 00000100:00000001:3.0:1788205166.430537:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321526535 00000100:00000001:3.0:1788205166.430539:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321525268 00000100:00000001:3.0:1788205166.430541:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321526535 00000100:00000001:3.0:1788205166.430543:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205166.430545:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.430547:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205166.430550:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205166/real 1788205166] req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205166.430557:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205166.430558:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205166.430559:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205166.430562:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205166.430563:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205166.430565:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205166.430567:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205166.430574:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430580:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205166.430581:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430583:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.430584:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205166.430587:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430593:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205166.430597:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430609:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205166.430612:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430613:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.430614:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430616:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205166.430618:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205166.430621:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205166.430624:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205166.430627:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205166.430631:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205166.430639:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519232:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205166.430643:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205166.430646:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430653:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205166.430654:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.430656:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205166.430658:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205166.430660:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205166.430663:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519360:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205166.430665:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205166.430667:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205166.430669:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.430671:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205166.430674:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205166.430676:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205166.430679:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205166.430681:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205166.430684:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205166.430687:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543519360, portal 10 00000100:00000001:3.0:1788205166.430689:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205166.430690:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904115840 : -109419805435776 : ffff9c7bb75d9880) 00000100:00000040:3.0:1788205166.430695:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430701:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205166.430703:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205166.430705:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205166.430707:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543519360, offset 0 00000400:00000200:3.0:1788205166.430710:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205166.430713:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205166.430716:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205166.430719:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205166.430722:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205166.430724:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205166.430726:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205166.430728:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471406:471406:256:4294967295] 192.168.201.118@tcp LPNI seq info [471265:471265:8:4294967295] 00000400:00000200:3.0:1788205166.430732:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205166.430737:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205166.430741:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.430744:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205166.430748:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205166.430750:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205166.430754:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205166.430757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205166.430760:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.430762:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205166.430765:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205166.430768:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205166.430771:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205166.430773:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205166.430775:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205166.430778:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430785:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.430788:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430794:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.430796:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205166.430799:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205166.430802:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205166.430805:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205166.430807:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321525269 00000100:00000001:3.0:1788205166.430809:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321526535 00000100:00000001:3.0:1788205166.430811:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321525269 00000100:00000001:3.0:1788205166.430814:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321526535 00000100:00000001:3.0:1788205166.430816:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205166.430818:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.430820:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205166.430823:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205166/real 1788205166] req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205166.430831:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205166.430833:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205166.430834:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205166.430837:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205166.430839:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205166.430841:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205166.430844:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205166.430852:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430860:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205166.430861:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430863:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.430864:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205166.430868:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430875:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205166.430878:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430885:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205166.430887:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430889:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.430890:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.430892:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205166.430894:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205166.430896:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205166.430899:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205166.430901:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205166.430918:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205166.430926:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519360:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205166.430930:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205166.430933:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430940:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205166.430942:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.430945:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205166.430947:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205166.430949:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205166.430952:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519488:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205166.430955:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205166.430957:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205166.430959:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.430960:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205166.430964:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205166.430966:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205166.430969:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205166.430972:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205166.430975:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205166.430978:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543519488, portal 4 00000100:00000001:3.0:1788205166.430984:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205166.430985:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904120320 : -109419805431296 : ffff9c7bb75daa00) 00000100:00000040:3.0:1788205166.430990:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.430997:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205166.430999:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205166.431001:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205166.431006:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543519488, offset 0 00000400:00000200:3.0:1788205166.431010:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205166.431014:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205166.431020:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205166.431023:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205166.431025:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205166.431028:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205166.431030:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205166.431033:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471407:471407:256:4294967295] 192.168.201.118@tcp LPNI seq info [471266:471266:8:4294967295] 00000400:00000200:3.0:1788205166.431037:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205166.431044:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205166.431049:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.431052:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205166.431056:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205166.431058:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205166.431063:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205166.431066:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205166.431069:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.431071:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205166.431074:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205166.431077:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205166.431080:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205166.431082:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205166.431084:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205166.431087:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431111:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.431114:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431120:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.431121:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205166.431124:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205166.431127:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205166.431130:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205166.431132:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321525279 00000100:00000001:3.0:1788205166.431134:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321526535 00000100:00000001:3.0:1788205166.431136:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321525279 00000100:00000001:3.0:1788205166.431139:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321526535 00000100:00000001:3.0:1788205166.431141:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205166.431143:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.431145:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205166.431148:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205166/real 1788205166] req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205166.431157:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205166.431158:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205166.431160:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205166.431163:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205166.431164:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205166.431167:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205166.431170:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205166.431178:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431184:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205166.431186:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431187:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.431188:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205166.431191:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431197:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205166.431200:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431206:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205166.431208:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431210:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.431211:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431213:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205166.431214:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205166.431216:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205166.431219:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205166.431221:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205166.431225:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205166.431231:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75daa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519488:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205166.431235:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205166.431238:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431243:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205166.431244:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.431246:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205166.431248:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205166.431250:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205166.431252:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519616:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205166.431255:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205166.431257:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205166.431258:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205166.431260:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205166.431263:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205166.431265:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205166.431267:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205166.431270:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205166.431272:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205166.431275:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543519616, portal 4 00000100:00000001:3.0:1788205166.431276:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205166.431277:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904116736 : -109419805434880 : ffff9c7bb75d9c00) 00000100:00000040:3.0:1788205166.431282:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431288:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205166.431290:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205166.431292:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205166.431294:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543519616, offset 0 00000400:00000200:3.0:1788205166.431299:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205166.431303:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205166.431305:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205166.431308:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205166.431310:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205166.431313:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205166.431315:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205166.431317:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471408:471408:256:4294967295] 192.168.201.118@tcp LPNI seq info [471267:471267:8:4294967295] 00000400:00000200:3.0:1788205166.431321:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205166.431326:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205166.431330:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.431333:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000800:00000200:3.0:1788205166.431337:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205166.431339:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:3.1:1788205166.431342:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205166.431345:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:3.0:1788205166.431349:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205166.431350:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:3.0:1788205166.431353:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205166.431356:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205166.431359:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:3.0:1788205166.431361:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:3.0:1788205166.431363:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205166.431366:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431371:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.431373:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205191 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431376:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.431377:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205166.431380:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205166.431382:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205166.431384:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205166.431386:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321525302 00000100:00000001:3.0:1788205166.431387:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321526535 00000100:00000001:3.0:1788205166.431389:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321525302 00000100:00000001:3.0:1788205166.431390:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321526535 00000100:00000001:3.0:1788205166.431392:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205166.431393:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.431394:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205166.431396:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205166/real 1788205166] req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205166.431401:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205166.431402:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205166.431403:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205166.431404:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205166.431405:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205166.431407:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205166.431409:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205166.431413:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431416:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205166.431417:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431418:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.431419:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205166.431421:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431424:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205166.431426:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431430:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205166.431432:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431433:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205166.431434:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431435:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205166.431436:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205166.431438:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205166.431439:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205166.431440:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205166.431442:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205166.431447:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519616:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205166.431449:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431450:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.431452:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75dbb80 x1875055543519232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431455:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205166.431457:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205166.431458:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205166.431459:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205166.431460:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205166.431461:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205166.431462:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205166.431463:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2ac00. 02000000:00000001:3.0:1788205166.431465:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205166.431466:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205166.431467:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dbb80. 00000100:00000001:3.0:1788205166.431469:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205166.431470:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431471:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.431472:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9880 x1875055543519360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431480:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205166.431481:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205166.431482:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205166.431484:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205166.431484:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205166.431485:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205166.431486:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205166.431487:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a800. 02000000:00000001:3.0:1788205166.431488:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205166.431488:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205166.431489:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9880. 00000100:00000001:3.0:1788205166.431490:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205166.431491:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431492:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.431493:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75daa00 x1875055543519488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431496:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205166.431497:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205166.431498:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205166.431498:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205166.431499:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205166.431500:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205166.431501:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205166.431501:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:3.0:1788205166.431502:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205166.431503:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205166.431503:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75daa00. 00000100:00000001:3.0:1788205166.431504:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205166.431505:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431506:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205166.431509:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9c00 x1875055543519616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205191 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205166.431515:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205166.431516:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205166.431517:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205166.431519:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205166.431520:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205166.431521:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205166.431523:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205166.431524:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:3.0:1788205166.431526:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205166.431527:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205166.431528:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9c00. 00000100:00000001:3.0:1788205166.431530:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205166.431531:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431533:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205166.431538:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205166.431539:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205166.431540:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205166.431542:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205167.260220:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205167.260227:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205167.260229:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205167.260233:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205167.260237:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205167.260239:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205167.260246:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205167.260248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205167.260249:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205167.260257:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1788205167.260258:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205167.260265:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205167.260266:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205167.260269:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205167.260272:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205167.260274:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205167.260278:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205167.260283:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205167.260954:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205167.260958:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205167.260963:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205167.260964:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205167.260967:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205167.260969:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205167.260972:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205167.260973:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205167.264798:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205167.264808:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205167.264813:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205167.772229:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321526535, at 4321526592 00000100:00000010:2.0:1788205167.772244:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc0c0. 00000100:00000040:3.0:1788205167.772314:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321526535, at 4321526592 00000100:00000010:3.0:1788205167.772328:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc720. 00000100:00000001:2.0:1788205168.284140:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205168.284147:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205168.284149:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205168.284153:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205168.284156:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205168.284159:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205168.284162:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205168.284165:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205168.284169:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205168.284207:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205168.284214:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205168.284216:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205168.284221:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205168.284224:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205168.284228:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205168.284231:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205168.284234:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205168.284237:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205168.284367:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205168.284371:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205168.284376:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205168.284377:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205168.284381:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205168.284382:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205168.284387:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205168.284388:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205168.287481:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205168.287489:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205168.287494:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205169.308198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205169.308202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205169.308207:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.308207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205169.308211:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205169.308214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205169.308216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205169.308218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.308222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205169.308224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205169.308228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.308228:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205169.308232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205169.308233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205169.308236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205169.308238:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205169.308243:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205169.308256:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205169.308261:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205169.308263:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205169.308266:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205169.308269:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205169.308272:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.308275:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205169.308287:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205169.308290:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205169.308504:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205169.308508:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205169.308511:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205170.332187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205170.332187:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205170.332190:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205170.332196:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205170.332197:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205170.332197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205170.332200:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205170.332201:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205170.332202:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205170.332208:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205170.332209:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205170.332210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205170.332215:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205170.332217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205170.332217:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205170.332218:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205170.332221:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205170.332222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205170.332223:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205170.332225:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205170.332227:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205170.332228:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205170.332229:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205170.332231:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205170.332234:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205170.332236:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205170.332237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205170.332241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205170.332245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205170.332247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205170.332252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205170.332253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205170.332538:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205170.332542:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205170.332544:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205171.356202:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205171.356202:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205171.356204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205171.356210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205171.356212:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205171.356212:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205171.356217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.356217:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205171.356219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205171.356223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205171.356225:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205171.356225:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205171.356228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205171.356231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205171.356232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205171.356233:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205171.356236:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205171.356239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205171.356239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.356239:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205171.356246:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205171.356249:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.356252:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205171.356254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.356257:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.356261:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205171.356297:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205171.356301:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205171.356305:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205171.548218:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205171.548228:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205171.548230:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205171.548234:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205171.548239:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205171.548247:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26857 00000400:00000200:0.0:1788205171.548264:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205171.548272:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205171.548275:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205171.548277:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548279:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205171.548281:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548282:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205171.548285:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205171.548286:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205171.548289:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205171.548291:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205171.548292:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205171.548294:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205171.548296:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548297:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205171.548298:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548299:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205171.548301:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205171.548303:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548305:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205171.548307:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205171.548309:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205171.548312:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548314:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205171.548316:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205171.548326:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788205171.548329:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205171.548332:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205171.548334:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548337:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205171.548339:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205171.548344:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205171.548352:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29400. 00000100:00000001:0.0:1788205171.548358:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205171.548367:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205171.548375:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205171.548378:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205171.548381:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205171.548397:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205171.548399:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205171.548401:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205171.548402:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205171.548404:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205171.548405:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205171.548407:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26857 00000400:00000200:0.0:1788205171.548410:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205171.548412:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205171.548413:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205171.548415:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548417:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205171.548418:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548420:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205171.548422:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205171.548423:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205171.548424:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205171.548425:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205171.548427:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205171.548428:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788205171.548429:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205171.548431:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205171.548432:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205171.548433:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548436:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205171.548438:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:3.0:1788205171.548438:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205171.548442:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205171.548442:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205171.548446:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205171.548447:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205171.548451:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205171.548453:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205171.548453:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205171.548457:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205171.548460:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205171.548465:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548467:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.548467:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205171.548470:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205171.548471:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788205171.548474:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205171.548476:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205171.548476:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788205171.548479:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548480:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205171.548480:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205171.548482:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205171.548486:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205171.548486:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205171.548492:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29c00. 00000100:00100000:3.0:1788205171.548495:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543519744:192.168.201.118@tcp:250:kworker.0 00000100:00000001:0.0:1788205171.548496:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205171.548503:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205171.548503:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205171.548509:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788205171.548510:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788205171.548512:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548513:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205171.548518:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1788205171.548519:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205171.548526:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205171.548526:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 00000100:00080000:0.0:1788205171.548530:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788205171.548531:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205171.548533:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205171.548536:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205171.548537:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205171.548538:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205171.548540:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205171.548542:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26857 00000400:00000010:3.0:1788205171.548545:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000200:0.0:1788205171.548546:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205171.548550:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000010:3.0:1788205171.548554:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00080000:0.0:1788205171.548558:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000200:3.0:1788205171.548561:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543519744, portal 25 00000100:00000040:0.0:1788205171.548565:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205171.548565:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205171.548567:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904121216 : -109419805430400 : ffff9c7bb75dad80) 00000100:00000001:0.0:1788205171.548570:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205171.548575:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205171.548575:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205171.548581:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205171.548587:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788205171.548588:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205171.548591:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000100:00000040:3.0:1788205171.548592:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000020:00000001:0.0:1788205171.548594:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:3.0:1788205171.548595:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734908. 00000100:00000200:3.0:1788205171.548599:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543519744, offset 0 00000020:00000040:0.0:1788205171.548600:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205171.548603:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205171.548604:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205171.548612:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:0.0:1788205171.548615:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205171.548619:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000040:0.0:1788205171.548622:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548625:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205171.548626:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1788205171.548628:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205171.548632:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:0.0:1788205171.548633:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205171.548636:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788205171.548638:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1788205171.548641:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:3.0:1788205171.548643:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548644:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205171.548647:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205171.548649:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471409:471409:256:4294967295] 192.168.201.118@tcp LPNI seq info [471268:471268:8:4294967295] 02000000:00000001:0.0:1788205171.548651:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205171.548654:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.548654:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205171.548659:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205171.548663:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788205171.548664:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000008:00000001:0.0:1788205171.548670:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205171.548673:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548676:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1788205171.548678:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd500. 00000100:00000010:0.0:1788205171.548682:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000800:00000200:3.0:1788205171.548686:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:0.0:1788205171.548687:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205171.548693:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000800:00000200:3.1:1788205171.548693:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 02000000:00000001:0.0:1788205171.548696:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548699:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205171.548700:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205171.548702:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205171.548706:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:3.0:1788205171.548709:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548710:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.548719:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205171.548722:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321531653 00000100:00100000:0.0:1788205171.548725:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:3.0:1788205171.548728:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc720. 00000100:00000001:0.0:1788205171.548736:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205171.548736:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:0.0:1788205171.548740:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.548742:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321531653 00000100:00000010:3.0:1788205171.548745:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc180. 00000100:00000040:0.0:1788205171.548747:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205171.548748:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205171.548752:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205171.548756:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.548761:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548763:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.548764:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788205171.548768:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205171.548772:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205171.548773:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788205171.548774:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205171.548777:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205171.548777:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788205171.548782:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788205171.548783:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205171.548786:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788205171.548789:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26857 00000100:00000001:3.0:1788205171.548790:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1788205171.548796:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205171.548797:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205171.548804:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205171.548808:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788205171.548809:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205171.548810:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205171.548831:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548834:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205171.548836:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.548838:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:3.0:1788205171.548838:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1788205171.548842:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205171.548845:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 02000000:00000001:3.0:1788205171.548845:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788205171.548847:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:3.0:1788205171.548851:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205171.548852:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788205171.548856:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00100000:3.0:1788205171.548858:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519872:192.168.201.118@tcp:38:kworker.0 00000020:00000001:0.0:1788205171.548860:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:3.0:1788205171.548866:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1788205171.548867:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:3.0:1788205171.548869:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788205171.548871:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788205171.548873:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205171.548875:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:3.0:1788205171.548877:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788205171.548878:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205171.548882:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 02000000:00000010:3.0:1788205171.548883:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 00000100:00080000:0.0:1788205171.548887:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000001:3.0:1788205171.548888:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548892:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205171.548897:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205171.548897:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:0.0:1788205171.548902:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:3.0:1788205171.548903:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77420. 02000000:00000001:0.0:1788205171.548907:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205171.548910:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000001:0.0:1788205171.548911:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205171.548916:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000200:3.0:1788205171.548916:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543519872, portal 10 00000100:00000001:3.0:1788205171.548921:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00000001:0.0:1788205171.548922:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.548923:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904122112 : -109419805429504 : ffff9c7bb75db100) 00000100:00000001:0.0:1788205171.548925:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205171.548930:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000100:00000040:3.0:1788205171.548933:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205171.548934:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205171.548939:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205171.548942:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.548946:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205171.548949:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788205171.548950:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205171.548953:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205171.548953:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205171.548959:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734e58. 02000000:00000010:0.0:1788205171.548960:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b800. 00000100:00000200:3.0:1788205171.548963:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543519872, offset 0 00000100:00000001:0.0:1788205171.548965:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.548970:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00100000:0.0:1788205171.548973:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205171.548978:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205171.548985:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:0.0:1788205171.548986:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788205171.548990:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788205171.548991:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.548995:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000040:0.0:1788205171.548999:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205171.549000:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205171.549007:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205171.549014:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471410:471410:255:4294967295] 192.168.201.118@tcp LPNI seq info [471269:471269:7:4294967295] 00000100:00000001:0.0:1788205171.549017:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.549021:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205171.549023:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205171.549028:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205171.549031:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788205171.549033:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:0.0:1788205171.549034:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205171.549041:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:3.0:1788205171.549043:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205171.549047:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd400. 00000800:00000200:3.0:1788205171.549051:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205171.549056:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:0.0:1788205171.549058:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26857 00000100:00000001:3.0:1788205171.549060:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205171.549065:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205171.549066:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:0.0:1788205171.549071:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000200:3.0:1788205171.549072:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205171.549076:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205171.549077:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321530386 00000100:00000001:3.0:1788205171.549082:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321531653 00000100:00000040:0.0:1788205171.549083:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205171.549087:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321530386 00000100:00000001:0.0:1788205171.549088:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205171.549114:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321531653 00000100:00000001:3.0:1788205171.549117:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205171.549119:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.549121:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205171.549126:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205171.549131:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205171.549136:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205171.549142:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:3.0:1788205171.549142:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1788205171.549145:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000001:3.0:1788205171.549146:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205171.549150:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000001:3.0:1788205171.549150:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1788205171.549154:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 02000000:00000001:3.0:1788205171.549154:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205171.549158:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205171.549159:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00100000:3.0:1788205171.549163:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520000:192.168.201.118@tcp:38:kworker.0 00000020:00000001:0.0:1788205171.549164:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205171.549169:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1788205171.549171:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:3.0:1788205171.549172:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205171.549175:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.549176:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788205171.549178:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1788205171.549181:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000010:3.0:1788205171.549183:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:0.0:1788205171.549184:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:3.0:1788205171.549187:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205171.549190:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000200:3.0:1788205171.549194:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205171.549196:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:3.0:1788205171.549198:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000100:00000001:0.0:1788205171.549202:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205171.549203:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 02000000:00000001:0.0:1788205171.549206:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:3.0:1788205171.549208:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543520000, portal 10 02000000:00000001:0.0:1788205171.549212:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205171.549212:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205171.549215:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904109568 : -109419805442048 : ffff9c7bb75d8000) 02000000:00000001:0.0:1788205171.549217:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.549220:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:3.0:1788205171.549223:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1788205171.549224:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205171.549230:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205171.549234:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.549240:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:0.0:1788205171.549241:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000100:00000040:3.0:1788205171.549244:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000020:00000040:0.0:1788205171.549246:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000400:00000010:3.0:1788205171.549248:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734198. 02000000:00000001:0.0:1788205171.549252:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000200:3.0:1788205171.549252:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543520000, offset 0 02000000:00000001:0.0:1788205171.549256:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.549260:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205171.549261:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205171.549264:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:3.0:1788205171.549266:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:0.0:1788205171.549267:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.549273:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 02000000:00000010:0.0:1788205171.549274:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2b000. 00000400:00000200:3.0:1788205171.549278:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1788205171.549280:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.549283:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205171.549288:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00100000:0.0:1788205171.549289:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205171.549291:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205171.549297:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471411:471411:254:4294967295] 192.168.201.118@tcp LPNI seq info [471270:471270:6:4294967295] 00000100:00000001:0.0:1788205171.549306:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788205171.549307:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205171.549312:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.549318:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000040:0.0:1788205171.549321:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788205171.549327:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205171.549333:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd100. 00000100:00000001:0.0:1788205171.549335:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205171.549340:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:0.0:1788205171.549342:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26868) 00000800:00000200:3.1:1788205171.549346:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205171.549350:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205171.549353:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205171.549356:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205171.549358:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321530388 00000100:00000001:3.0:1788205171.549361:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321531653 00000100:00000001:3.0:1788205171.549363:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321530388 00000100:00000001:3.0:1788205171.549365:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321531653 00000100:00000001:3.0:1788205171.549367:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205171.549368:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.549371:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.549372:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.549375:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205171.549377:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788205171.549379:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205171.549380:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205171.549381:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205171.549385:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205171.549391:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205171.549393:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205171.549395:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205171.549397:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205171.549399:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205171.549402:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520128:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205171.549405:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205171.549406:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205171.549407:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205171.549409:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205171.549412:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205171.549413:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.549416:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205171.549419:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77a50. 00000400:00000010:3.0:1788205171.549422:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734d48. 00000100:00000200:3.0:1788205171.549424:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543520128, portal 4 00000100:00000001:3.0:1788205171.549426:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205171.549427:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904110464 : -109419805441152 : ffff9c7bb75d8380) 00000100:00000040:3.0:1788205171.549433:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205171.549439:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205171.549441:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205171.549443:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734088. 00000100:00000200:3.0:1788205171.549445:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543520128, offset 0 00000400:00000200:3.0:1788205171.549448:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205171.549451:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205171.549455:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205171.549457:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205171.549459:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205171.549461:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205171.549463:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205171.549466:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471412:471412:253:4294967295] 192.168.201.118@tcp LPNI seq info [471271:471271:5:4294967295] 00000400:00000200:3.0:1788205171.549470:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205171.549475:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205171.549479:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205171.549482:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205171.549485:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205171.549488:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205171.549491:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205171.549493:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205171.549495:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205171.549497:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321530397 00000100:00000001:3.0:1788205171.549499:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321531653 00000100:00000001:3.0:1788205171.549501:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321530397 00000100:00000001:3.0:1788205171.549503:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321531653 00000100:00000001:3.0:1788205171.549505:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205171.549507:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.549509:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205171.549512:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205171.549588:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205171.549590:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205171.549592:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205171.549594:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205171.549596:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205171.549599:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520256:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205171.549602:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205171.549604:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205171.549605:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205171.549607:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205171.549611:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205171.549612:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205171.549615:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205171.549618:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb771b8. 00000400:00000010:3.0:1788205171.549621:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734bb0. 00000100:00000200:3.0:1788205171.549624:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543520256, portal 4 00000100:00000001:3.0:1788205171.549626:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205171.549627:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904111360 : -109419805440256 : ffff9c7bb75d8700) 00000100:00000040:3.0:1788205171.549632:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205171.549638:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205171.549640:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205171.549643:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987344c8. 00000100:00000200:3.0:1788205171.549645:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543520256, offset 0 00000400:00000200:3.0:1788205171.549650:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205171.549655:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205171.549659:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205171.549662:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205171.549664:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205171.549666:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205171.549669:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205171.549672:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471413:471413:252:4294967295] 192.168.201.118@tcp LPNI seq info [471272:471272:4:4294967295] 00000400:00000200:3.0:1788205171.549676:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205171.549682:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205171.549686:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205171.549690:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd900. 00000800:00000200:3.0:1788205171.549693:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205171.549697:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205171.549700:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205171.549702:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205171.549705:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205171.549707:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321530420 00000100:00000001:3.0:1788205171.549710:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321531653 00000100:00000001:3.0:1788205171.549712:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321530420 00000100:00000001:3.0:1788205171.549713:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321531653 00000100:00000001:3.0:1788205171.549715:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205171.549717:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.549720:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.549722:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205171.549725:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205171.549726:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788205171.549728:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205171.549729:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205171.549731:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205171.549733:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205172.380325:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205172.380336:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205172.380339:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205172.380347:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205172.380351:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205172.380355:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205172.380370:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205172.380373:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205172.380377:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205172.380577:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205172.380583:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205172.380590:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205172.380591:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205172.380594:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205172.380595:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205172.380598:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205172.380599:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205172.380740:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205172.380743:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205172.380746:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205172.381877:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205172.381882:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205172.381884:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205172.381888:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205172.381891:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205172.381894:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205172.381896:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205172.381899:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205172.381901:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205172.892702:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321531653, at 4321531712 00000100:00000040:2.0:1788205172.892732:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321531653, at 4321531712 00000100:00000010:2.0:1788205172.892748:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc720. 00000100:00000010:3.0:1788205172.892820:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc180. 00000100:00000001:2.0:1788205173.404257:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205173.404262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205173.404267:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205173.404269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205173.404272:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205173.404282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205173.404282:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205173.404285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205173.404289:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205173.404292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205173.404295:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205173.404296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205173.404301:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205173.404304:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205173.404305:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205173.404308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205173.404312:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205173.404366:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205173.404371:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205173.404374:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205173.404901:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205173.404904:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205173.404905:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205173.404907:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205173.404908:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205173.404910:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205173.404911:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205173.404913:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205173.404914:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205174.428171:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205174.428181:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205174.428183:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205174.428188:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205174.428193:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205174.428197:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205174.428200:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205174.428204:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205174.428207:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205174.428218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205174.428222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205174.428225:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205174.428226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205174.428228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205174.428229:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205174.428231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205174.428232:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205174.428271:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205174.428275:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205174.428278:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205174.428937:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205174.428944:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205174.428947:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205174.428952:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205174.428956:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205174.428960:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205174.428963:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205174.428966:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205174.428969:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788205174.623134:0:256120:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788205174.623178:0:256120:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788205174.623187:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205174.623191:0:256120:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788205174.623197:0:256120:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205174.623202:0:256120:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788205174.623206:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205174.623211:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd500. 00000400:00000200:2.0:1788205174.623217:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205174.623221:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205174.623228:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734908 00000400:00000010:2.0:1788205174.623230:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734908. 00000100:00000001:2.0:1788205174.623236:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205174.623244:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205174.623266:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205174.623270:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205174.623277:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205174.623278:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205174.623288:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205174.623290:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd400. 00000400:00000200:2.0:1788205174.623294:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205174.623298:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205174.623301:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734e58 00000400:00000010:2.0:1788205174.623303:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734e58. 00000100:00000001:2.0:1788205174.623306:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1788205174.623308:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1788205174.623312:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623315:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205174.623321:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788205174.623324:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205174.623333:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205174.623333:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205171/real 1788205174] req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205174.623352:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205174.623356:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:3.0:1788205174.623360:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000100:2.0:1788205174.623364:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000400:00000001:3.0:1788205174.623364:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000010:2.0:1788205174.623369:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd100. 00000400:00000010:3.0:1788205174.623371:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:2.0:1788205174.623373:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1788205174.623377:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000200:2.0:1788205174.623379:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000010:3.0:1788205174.623382:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205174.623387:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1788205174.623390:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734198 00000100:00000200:3.0:1788205174.623392:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788205174.623393:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734198. 00000100:00000001:2.0:1788205174.623399:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205174.623406:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205174.623411:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205174.623432:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205174.623434:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205174.623438:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205174.623439:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623444:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.623447:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205174.623450:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205174.623453:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1788205174.623457:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1788205174.623463:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205174.623469:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:2.0:1788205174.623474:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:3.0:1788205174.623475:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1788205174.623480:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205174.623484:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734088 00000100:00000040:3.0:1788205174.623485:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788205174.623487:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734088. 00000100:00000001:2.0:1788205174.623492:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205174.623499:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623500:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205174.623506:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205174.623512:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205174.623516:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623519:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.623523:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623528:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205174.623529:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205174.623532:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205174.623532:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000800:00000100:2.0:1788205174.623537:0:256120:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000100:00080000:3.0:1788205174.623540:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000800:00000010:2.0:1788205174.623541:0:256120:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd900. 00000400:00000200:2.0:1788205174.623545:0:256120:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00080000:3.0:1788205174.623550:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000400:00000200:2.0:1788205174.623551:0:256120:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205174.623558:0:256120:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987344c8 00000100:00000001:3.0:1788205174.623558:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000010:2.0:1788205174.623560:0:256120:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987344c8. 00000100:00000001:2.0:1788205174.623564:0:256120:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1788205174.623567:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788205174.623570:0:256120:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205174.623593:0:256120:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1788205174.623595:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543519744:192.168.201.118@tcp:250:kworker.0 00000100:00000040:2.0:1788205174.623599:0:256120:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205196 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623605:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205174.623610:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788205174.623611:0:256120:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205174.623614:0:256120:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1788205174.623618:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75dad80 x1875055543519744/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623626:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205174.623630:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205174.623634:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205174.623639:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205174.623641:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205174.623644:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205174.623646:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205174.623649:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29400. 02000000:00000001:3.0:1788205174.623652:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205174.623654:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205174.623656:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dad80. 00000100:00000001:3.0:1788205174.623660:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205174.623661:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623663:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205174.623665:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205174.623666:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205174.623667:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205174.623672:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205171/real 1788205174] req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205174.623680:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205174.623682:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205174.623684:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77a50. 00000400:00000200:3.0:1788205174.623686:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734d48 00000400:00000010:3.0:1788205174.623688:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734d48. 00000100:00000001:3.0:1788205174.623690:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205174.623694:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205174.623702:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623708:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205174.623710:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623711:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.623713:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205174.623717:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623723:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205174.623727:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623739:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205174.623742:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623743:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.623745:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623746:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205174.623748:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205174.623751:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205174.623755:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205174.623757:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205174.623794:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205174.623803:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520128:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205174.623808:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205174.623812:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205171/real 1788205174] req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205174.623820:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205174.623821:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205174.623823:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb771b8. 00000400:00000200:3.0:1788205174.623825:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734bb0 00000400:00000010:3.0:1788205174.623827:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734bb0. 00000100:00000001:3.0:1788205174.623829:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205174.623833:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205174.623841:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623847:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205174.623849:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623851:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.623852:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205174.623856:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623862:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205174.623865:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623872:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205174.623874:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623875:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.623876:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623878:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205174.623880:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205174.623882:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205174.623886:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205174.623888:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205174.623891:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205174.623899:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520256:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205174.623903:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205174.623907:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205171/real 1788205174] req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205174.623915:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205174.623916:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205174.623918:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77420. 00000400:00000200:3.0:1788205174.623921:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205174.623922:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205174.623925:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205174.623928:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205174.623937:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623943:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205174.623944:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623946:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.623947:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205174.623950:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623957:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205174.623960:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.623966:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205174.623968:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623970:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.623971:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.623973:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205174.623975:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205174.623977:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205174.623981:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205174.623983:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205174.623987:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205174.623994:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543519872:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205174.623998:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205174.624002:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205171/real 1788205174] req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205174.624009:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205174.624011:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205174.624012:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205174.624015:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205174.624017:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205174.624019:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205174.624022:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205174.624030:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.624036:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205174.624038:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.624039:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.624041:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205174.624044:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.624050:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205174.624054:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.624059:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205174.624061:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.624063:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205174.624064:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.624066:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205174.624068:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205174.624071:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205174.624074:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205174.624077:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205174.624081:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205174.626014:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520000:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205174.626021:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.626024:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205174.626029:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8380 x1875055543520128/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.626038:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205174.626041:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205174.626045:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205174.626048:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205174.626051:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205174.626052:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205174.626055:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205174.626057:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b800. 02000000:00000001:3.0:1788205174.626060:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205174.626062:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205174.626063:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8380. 00000100:00000001:3.0:1788205174.626067:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205174.626068:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.626070:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205174.626073:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8700 x1875055543520256/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.626079:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205174.626080:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205174.626082:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205174.626084:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205174.626085:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205174.626086:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205174.626088:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205174.626105:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2b000. 02000000:00000001:3.0:1788205174.626108:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205174.626109:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205174.626111:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8700. 00000100:00000001:3.0:1788205174.626113:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205174.626115:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.626117:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205174.626122:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db100 x1875055543519872/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.626218:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205174.626219:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205174.626221:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205174.626224:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205174.626226:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205174.626227:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205174.626229:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205174.626230:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29c00. 02000000:00000001:3.0:1788205174.626233:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205174.626234:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205174.626236:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db100. 00000100:00000001:3.0:1788205174.626238:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205174.626240:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.626242:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205174.626246:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8000 x1875055543520000/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205196 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205174.626253:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205174.626254:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205174.626256:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205174.626258:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205174.626259:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205174.626261:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205174.626263:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205174.626264:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28800. 02000000:00000001:3.0:1788205174.626266:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205174.626268:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205174.626269:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8000. 00000100:00000001:3.0:1788205174.626271:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205174.626273:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.626276:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205174.626283:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205174.626285:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205174.626288:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205174.626290:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205175.452259:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205175.452271:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205175.452275:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205175.452282:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205175.452287:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205175.452291:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205175.452294:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205175.452297:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205175.452301:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452543:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205175.452547:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452552:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205175.452556:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205175.452558:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205175.452563:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452566:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205175.452569:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452571:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452574:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452577:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205175.452579:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205175.452581:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205175.452582:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452584:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205175.452585:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452587:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452589:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452593:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205175.452594:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205175.452596:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205175.452597:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452598:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205175.452599:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452602:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205175.452604:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205175.452724:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205175.452731:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205175.452733:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205175.452739:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205175.452743:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205175.452773:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205175.452778:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205175.452780:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205175.452783:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205175.454578:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205175.454584:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205175.454589:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205176.476233:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205176.476241:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205176.476243:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205176.476247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205176.476249:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205176.476254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205176.476262:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205176.476264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205176.476268:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205176.476269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205176.476277:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205176.476279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205176.476282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205176.476282:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205176.476289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205176.476292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205176.476296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205176.478109:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205176.478116:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205176.478121:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205176.478124:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205176.478131:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205176.478133:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205176.478138:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205176.478142:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205176.478145:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205176.478159:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205176.478162:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205176.478165:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205176.668269:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205176.668348:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205176.668352:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205176.668357:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205176.668363:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205176.668372:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26863 00000400:00000200:3.0:1788205176.668384:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205176.668393:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205176.668397:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205176.668401:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.668404:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.668406:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.668409:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205176.668414:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205176.668416:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205176.668419:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205176.668423:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205176.668426:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205176.668429:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205176.668433:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.668434:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.668437:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.668439:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205176.668442:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205176.668446:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205176.668449:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205176.668453:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205176.668457:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205176.668474:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.668477:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205176.668480:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205176.668491:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205176.668496:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205176.668500:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205176.668502:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.668506:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205176.668510:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205176.668515:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205176.668525:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788205176.668533:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205176.668544:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205176.668555:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205176.671282:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205176.671513:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.671620:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205176.671626:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205176.671630:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205176.671638:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.671648:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205176.671650:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.671655:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205176.671660:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205176.671663:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205176.671667:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543520384:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205176.671682:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205176.671688:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205176.671692:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.671695:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205176.671707:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205176.671719:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205176.671737:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205176.671743:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205176.671754:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205176.671764:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543520384, portal 25 00000100:00000001:1.0:1788205176.671766:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205176.671769:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788205176.671776:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.671785:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205176.671787:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205176.671790:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205176.671795:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543520384, offset 0 00000400:00000200:1.0:1788205176.671803:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205176.671814:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.671825:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205176.671832:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205176.671836:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205176.671841:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205176.671844:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205176.671850:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471414:471414:256:4294967295] 192.168.201.118@tcp LPNI seq info [471273:471273:8:4294967295] 00000400:00000200:1.0:1788205176.671855:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.671867:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205176.671875:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.671882:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205176.671890:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205176.671894:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205176.671901:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205176.671905:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205176.671910:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.671913:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205176.671917:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205176.671922:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205176.671930:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205176.671932:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205176.671938:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205176.671943:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205176.671972:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205176.671977:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205176.671982:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205176.671984:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205176.671988:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205176.671992:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205176.671995:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26863 00000400:00000200:3.0:1788205176.672000:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205176.672005:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205176.672008:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205176.672011:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.672015:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.672017:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.672019:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205176.672023:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205176.672025:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205176.672028:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205176.672031:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205176.672034:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205176.672036:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205176.672039:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.672041:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.672043:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.672044:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205176.672047:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205176.672050:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205176.672053:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205176.672055:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205176.672063:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205176.672066:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.672068:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205176.672073:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205176.672076:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.672079:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205176.672086:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788205176.672109:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205176.672115:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205176.672117:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.672121:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205176.672124:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205176.672129:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205176.672136:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:3.0:1788205176.672144:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205176.672151:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205176.672160:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205176.672164:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205176.672167:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.675947:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.675971:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.675985:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.675988:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205176.676010:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205176.676018:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205176.676023:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321536777 00000100:00000010:1.0:1788205176.676034:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647c00. 00000100:00000001:1.0:1788205176.676043:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205176.676051:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321536777 00000100:00000010:1.0:1788205176.676055:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647240. 00000100:00000001:1.0:1788205176.676058:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205176.676062:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205176.676066:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.676070:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205176.676075:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205176/real 1788205176] req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205176.676088:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205176.676137:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205176.676143:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205176.676147:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205176.676150:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205176.676154:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205176.676159:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205176.676168:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.676176:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205176.676177:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.676179:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.676181:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205176.676185:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.676192:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205176.676197:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.676204:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205176.676207:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.676209:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.676210:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.676213:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205176.676216:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205176.676220:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205176.676225:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205176.676228:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205176.676232:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205176.676241:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543520384:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205176.676246:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.676249:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.676252:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543520384/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.676257:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205176.676261:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205176.676265:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205176.676270:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205176.676273:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205176.676274:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205176.676277:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205176.676280:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205176.676284:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205176.676287:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205176.676289:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205176.676426:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205176.676428:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.676431:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.676440:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205176.676442:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.676445:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205176.676448:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205176.676450:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205176.676456:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205176.676556:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205176.676562:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205176.676567:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205176.676569:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205176.676571:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205176.676575:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205176.676580:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26863 00000400:00000200:3.0:1788205176.676587:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205176.676593:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205176.676597:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205176.676600:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676604:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.676606:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676609:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205176.676612:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205176.676615:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205176.676619:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205176.676622:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205176.676625:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205176.676628:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205176.676631:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676633:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.676635:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676636:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205176.676640:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205176.676643:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676646:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205176.676648:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205176.676656:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205176.676659:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676662:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205176.676667:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205176.676670:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676673:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205176.676680:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205176.676684:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205176.676689:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205176.676690:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676694:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205176.676697:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205176.676702:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205176.676710:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788205176.676717:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205176.676725:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205176.676745:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205176.676751:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205176.676756:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205176.676766:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205176.676769:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205176.676773:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205176.676774:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205176.676776:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205176.676779:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205176.676782:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26863 00000400:00000200:3.0:1788205176.676786:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205176.676789:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205176.676791:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205176.676794:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676796:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.676798:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676800:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205176.676802:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205176.676803:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205176.676805:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205176.676807:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205176.676809:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205176.676811:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205176.676814:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676815:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.676817:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676819:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205176.676822:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205176.676824:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676827:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205176.676829:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205176.676832:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205176.676834:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676836:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205176.676839:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205176.676841:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676843:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205176.676847:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788205176.676849:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205176.676851:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205176.676852:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676854:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205176.676855:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205176.676857:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205176.676861:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788205176.676865:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205176.676869:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205176.676877:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205176.676879:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205176.676882:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205176.676889:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205176.676892:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205176.676895:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205176.676896:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205176.676897:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205176.676900:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205176.676902:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26863 00000400:00000200:3.0:1788205176.676905:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205176.676908:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205176.676910:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205176.676913:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676915:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.676917:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676918:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205176.676920:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205176.676921:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205176.676923:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205176.676924:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205176.676926:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205176.676928:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205176.676931:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676933:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205176.676934:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676936:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205176.676938:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205176.676941:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205176.676943:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205176.676945:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205176.676948:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205176.676949:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676952:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205176.676954:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205176.676956:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676958:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205176.676963:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205176.676965:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205176.676968:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205176.676969:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205176.676970:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205176.676972:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205176.676974:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205176.676977:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788205176.676981:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205176.676986:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205176.676993:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205176.676995:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205176.676998:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205176.677005:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205176.677010:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26873) 00000100:00000001:1.0:1788205176.677154:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205176.677157:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.677161:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205176.677164:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205176.677166:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205176.677170:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520512:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205176.677175:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205176.677180:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205176.677183:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.677186:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205176.677190:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205176.677192:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205176.677198:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205176.677201:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205176.677208:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205176.677214:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543520512, portal 10 00000100:00000001:1.0:1788205176.677216:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205176.677218:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205176.677225:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677231:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205176.677233:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205176.677236:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205176.677239:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543520512, offset 0 00000400:00000200:1.0:1788205176.677244:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205176.677252:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.677259:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205176.677264:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205176.677268:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205176.677272:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205176.677275:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205176.677279:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471415:471415:256:4294967295] 192.168.201.118@tcp LPNI seq info [471274:471274:8:4294967295] 00000400:00000200:1.0:1788205176.677283:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.677356:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205176.677362:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.677368:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205176.677375:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205176.677379:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205176.677386:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205176.677390:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205176.677394:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.677397:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205176.677401:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205176.677406:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205176.677412:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205176.677414:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205176.677418:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205176.677422:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677429:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.677432:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677437:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.677439:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205176.677444:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205176.677448:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205176.677452:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205176.677454:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321535515 00000100:00000001:1.0:1788205176.677457:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321536777 00000100:00000001:1.0:1788205176.677520:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321535515 00000100:00000001:1.0:1788205176.677524:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321536777 00000100:00000001:1.0:1788205176.677527:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205176.677529:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.677531:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205176.677536:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205176/real 1788205176] req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205176.677544:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205176.677545:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205176.677547:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205176.677551:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205176.677552:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205176.677555:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205176.677558:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205176.677566:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677572:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205176.677573:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677575:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.677576:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205176.677579:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677586:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205176.677589:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677596:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205176.677599:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677600:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.677602:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677604:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205176.677606:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205176.677608:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205176.677612:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205176.677614:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205176.677618:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205176.677626:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520512:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205176.677630:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677632:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.677635:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543520512/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677640:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205176.677642:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205176.677644:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205176.677647:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205176.677648:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205176.677650:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205176.677652:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205176.677653:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205176.677656:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205176.677657:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205176.677659:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205176.677662:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205176.677663:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677666:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677670:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205176.677672:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.677674:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205176.677675:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205176.677677:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205176.677682:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677688:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205176.677689:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.677691:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205176.677693:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205176.677695:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205176.677698:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520640:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205176.677701:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205176.677703:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205176.677705:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.677706:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205176.677710:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205176.677711:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205176.677714:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205176.677717:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205176.677720:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205176.677723:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543520640, portal 10 00000100:00000001:1.0:1788205176.677737:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205176.677739:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205176.677745:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677752:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205176.677754:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205176.677757:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205176.677759:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543520640, offset 0 00000400:00000200:1.0:1788205176.677763:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205176.677768:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.677772:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205176.677775:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205176.677777:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205176.677780:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205176.677782:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205176.677786:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471416:471416:256:4294967295] 192.168.201.118@tcp LPNI seq info [471275:471275:8:4294967295] 00000400:00000200:1.0:1788205176.677789:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.677796:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205176.677800:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.677804:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205176.677808:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205176.677811:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205176.677814:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205176.677817:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205176.677820:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.677822:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205176.677824:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205176.677827:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205176.677830:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205176.677832:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205176.677835:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205176.677838:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677846:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.677849:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677854:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.677855:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205176.677858:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205176.677861:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205176.677864:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205176.677866:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321535516 00000100:00000001:1.0:1788205176.677868:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321536777 00000100:00000001:1.0:1788205176.677870:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321535516 00000100:00000001:1.0:1788205176.677872:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321536777 00000100:00000001:1.0:1788205176.677874:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205176.677875:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.677877:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205176.677880:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205176/real 1788205176] req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205176.677887:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205176.677889:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205176.677890:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205176.677892:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205176.677894:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205176.677896:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205176.677898:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205176.677905:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677911:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205176.677912:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677913:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.677914:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205176.677918:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677923:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205176.677927:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677933:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205176.677936:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677938:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.677940:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.677941:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205176.677944:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205176.677947:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205176.677950:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205176.677952:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205176.677956:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205176.677963:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520640:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205176.677966:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205176.677969:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.677975:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205176.677977:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.677980:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205176.677982:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205176.677983:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205176.677986:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520768:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205176.677989:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205176.677991:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205176.677992:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.677994:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205176.677997:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205176.677999:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205176.678003:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205176.678005:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205176.678008:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205176.678011:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543520768, portal 4 00000100:00000001:1.0:1788205176.678013:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205176.678015:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788205176.678020:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.678027:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205176.678029:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205176.678031:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205176.678034:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543520768, offset 0 00000400:00000200:1.0:1788205176.678037:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205176.678042:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.678045:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205176.678048:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205176.678050:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205176.678053:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205176.678055:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205176.678058:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471417:471417:256:4294967295] 192.168.201.118@tcp LPNI seq info [471276:471276:8:4294967295] 00000400:00000200:1.0:1788205176.678062:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.678068:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205176.678072:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.678075:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205176.678079:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205176.678082:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205176.678086:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205176.679407:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205176.679423:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.679427:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205176.679433:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205176.679438:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205176.679444:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205176.679447:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205176.679452:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205176.679459:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679531:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.679536:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679543:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.679545:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205176.679552:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205176.679555:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205176.679558:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205176.679561:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321535527 00000100:00000001:1.0:1788205176.679563:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321536777 00000100:00000001:1.0:1788205176.679565:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321535527 00000100:00000001:1.0:1788205176.679567:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321536777 00000100:00000001:1.0:1788205176.679568:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205176.679570:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.679572:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205176.679574:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205176/real 1788205176] req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205176.679581:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205176.679583:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205176.679584:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205176.679586:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205176.679587:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205176.679588:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205176.679590:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205176.679596:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679600:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205176.679601:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.679602:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.679604:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205176.679606:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679611:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205176.679614:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679619:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205176.679621:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.679622:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.679623:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.679625:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205176.679627:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205176.679630:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205176.679634:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205176.679637:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205176.679641:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205176.679650:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520768:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205176.679654:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205176.679658:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679665:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205176.679667:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.679670:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205176.679673:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205176.679676:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205176.679680:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520896:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205176.679684:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205176.679689:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205176.679690:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205176.679692:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205176.679699:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205176.679700:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205176.679705:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205176.679708:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205176.679712:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205176.679716:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543520896, portal 4 00000100:00000001:1.0:1788205176.679718:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205176.679719:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788205176.679738:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679745:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205176.679747:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205176.679750:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205176.679753:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543520896, offset 0 00000400:00000200:1.0:1788205176.679758:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205176.679764:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.679769:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205176.679772:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205176.679776:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205176.679780:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205176.679782:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205176.679786:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471418:471418:256:4294967295] 192.168.201.118@tcp LPNI seq info [471277:471277:8:4294967295] 00000400:00000200:1.0:1788205176.679790:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205176.679797:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205176.679802:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.679806:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205176.679812:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205176.679815:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1788205176.679819:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205176.679822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1788205176.679826:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205176.679828:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205176.679831:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205176.679834:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205176.679837:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205176.679839:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205176.679842:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205176.679846:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679853:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.679856:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205201 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679862:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.679863:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205176.679866:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205176.679870:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205176.679873:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205176.679875:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321535550 00000100:00000001:1.0:1788205176.679877:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321536777 00000100:00000001:1.0:1788205176.679879:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321535550 00000100:00000001:1.0:1788205176.679881:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321536777 00000100:00000001:1.0:1788205176.679884:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205176.679885:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.679887:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205176.679891:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205176/real 1788205176] req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205176.679899:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205176.679900:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205176.679902:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205176.679905:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205176.679907:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205176.679909:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205176.679913:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205176.679921:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679927:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205176.679928:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.679930:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.679931:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205176.679934:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679941:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205176.679944:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679951:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205176.679953:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.679955:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205176.679956:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.679958:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205176.679960:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205176.679962:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205176.679966:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205176.679969:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205176.679973:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205176.679981:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543520896:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205176.679985:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.679987:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.679991:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543520640/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.679997:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205176.680000:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205176.680002:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205176.680005:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205176.680007:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205176.680009:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205176.680011:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205176.680014:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205176.680018:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205176.680020:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205176.680022:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205176.680025:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205176.680026:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.680028:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.680031:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543520768/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.680038:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205176.680039:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205176.680041:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205176.680043:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205176.680044:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205176.680045:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205176.680047:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205176.680048:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205176.680050:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205176.680051:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205176.680052:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205176.680055:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205176.680056:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.680058:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205176.680061:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543520896/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205201 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205176.680067:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205176.680069:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205176.680071:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205176.680073:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205176.680074:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205176.680076:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205176.680077:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205176.680079:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205176.680081:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205176.680082:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205176.680084:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205176.680087:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205176.680088:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.680111:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205176.680119:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205176.680121:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205176.680123:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205176.680125:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205177.500254:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205177.500254:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205177.500255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205177.500263:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205177.500264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205177.500266:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205177.500269:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205177.500271:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205177.500275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205177.500276:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205177.500278:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205177.500281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205177.500286:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205177.500287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205177.500294:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205177.500295:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205177.500297:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205177.500302:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205177.500304:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205177.500305:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205177.500307:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205177.500309:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205177.500315:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205177.500316:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205177.500324:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205177.500329:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205177.500350:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205177.500355:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205177.500358:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205178.012214:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321536777, at 4321536832 00000100:00000010:2.0:1788205178.012237:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c00. 00000100:00000040:3.0:1788205178.012427:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321536777, at 4321536832 00000100:00000010:3.0:1788205178.012433:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647240. 00010000:00000001:0.0:1788205178.524228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205178.524236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205178.524242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205178.524243:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205178.524244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205178.524247:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205178.524249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205178.524250:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205178.524254:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205178.524258:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205178.524261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205178.524263:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205178.524264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205178.524270:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205178.524274:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205178.524278:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205178.524281:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205178.524353:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205178.524361:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205178.524367:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205178.524774:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205178.524778:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205178.524780:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205178.524782:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205178.524784:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205178.524786:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205178.524787:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205178.524789:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205178.524791:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205179.548264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205179.548275:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205179.548277:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205179.548282:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205179.548288:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205179.548291:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205179.548293:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205179.548296:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205179.548301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205179.548303:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205179.548306:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205179.548308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205179.548311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205179.548313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205179.548494:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205179.548499:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205179.548501:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205179.548506:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205179.548510:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205179.548513:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205179.548517:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205179.548519:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205179.548523:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205179.548736:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205179.548741:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205179.548745:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205179.548761:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205179.548767:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205179.548769:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205179.548772:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205179.548774:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205179.548777:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205179.548780:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205179.548782:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205179.548784:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205180.572229:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205180.572233:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205180.572239:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205180.572241:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205180.572242:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.572245:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205180.572248:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205180.572249:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205180.572254:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205180.572254:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205180.572260:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.572260:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205180.572266:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205180.572266:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205180.572272:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.572272:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205180.572279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205180.572280:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205180.572629:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205180.572632:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205180.572636:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205180.572637:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205180.572639:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205180.572640:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205180.572642:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205180.572643:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205180.572670:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205180.572673:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205180.572676:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205181.596185:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205181.596189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205181.596189:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205181.596194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205181.596196:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205181.596196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205181.596200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205181.596201:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205181.596205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205181.596206:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205181.596208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205181.596211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205181.596212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.596214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205181.596218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205181.596218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205181.596220:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205181.596221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205181.596222:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205181.596224:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205181.596227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205181.596230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205181.596230:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205181.596234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205181.596235:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205181.596240:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205181.596286:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205181.596291:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205181.596294:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205181.788254:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205181.788266:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205181.788268:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205181.788272:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205181.788276:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205181.788283:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26868 00000400:00000200:3.0:1788205181.788294:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205181.788301:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205181.788304:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205181.788307:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788309:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.788311:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788313:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205181.788317:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205181.788319:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205181.788321:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205181.788325:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205181.788326:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205181.788328:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205181.788330:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788331:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.788332:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788333:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205181.788335:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205181.788337:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788339:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205181.788342:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205181.788346:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205181.788349:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788352:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205181.788354:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205181.788367:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205181.788372:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205181.788376:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205181.788378:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788382:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205181.788385:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205181.788390:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205181.788400:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413800. 00000100:00000001:3.0:1788205181.788408:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205181.788418:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205181.788428:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205181.788432:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205181.788436:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205181.788467:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205181.788471:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205181.788475:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205181.788477:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205181.788478:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205181.788481:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205181.788484:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26868 00000400:00000200:3.0:1788205181.788488:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205181.788491:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205181.788493:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205181.788496:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788498:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.788500:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788502:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205181.788504:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205181.788506:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205181.788508:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205181.788511:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205181.788513:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205181.788516:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205181.788519:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788522:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205181.788524:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1788205181.788524:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788529:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205181.788531:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1788205181.788532:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:1.0:1788205181.788537:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788205181.788537:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788542:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205181.788544:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205181.788549:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205181.788552:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205181.788557:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788562:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205181.788568:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788205181.788568:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788205181.788571:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788205181.788574:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.788578:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205181.788579:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.788583:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1788205181.788586:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 02000000:00000001:1.0:1788205181.788587:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205181.788591:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00100000:1.0:1788205181.788595:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543521024:192.168.201.118@tcp:250:kworker.0 02000000:00000001:3.0:1788205181.788596:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205181.788598:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788600:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205181.788603:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205181.788606:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205181.788606:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205181.788612:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:1.0:1788205181.788615:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205181.788615:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.788620:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205181.788622:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205181.788625:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788205181.788631:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205181.788633:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205181.788635:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000040:3.0:1788205181.788636:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205181.788659:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788671:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205181.788675:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205181.788679:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205181.788681:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205181.788682:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205181.788685:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205181.788688:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26868 00000400:00000200:3.0:1788205181.788692:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205181.788695:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205181.788697:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205181.788700:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788702:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.788704:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788707:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205181.788709:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205181.788710:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205181.788712:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205181.788714:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205181.788716:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205181.788718:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205181.788721:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788723:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.788725:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788727:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205181.788729:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205181.788732:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788734:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205181.788736:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205181.788739:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205181.788741:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788743:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205181.788745:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205181.788748:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788749:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205181.788753:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205181.788755:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205181.788757:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205181.788758:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788759:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205181.788760:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205181.788761:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205181.788764:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:3.0:1788205181.788767:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205181.788771:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205181.788776:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205181.788778:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205181.788780:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205181.788784:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205181.788786:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205181.788789:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:1.0:1788205181.788790:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205181.788790:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205181.788793:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205181.788796:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205181.788799:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26868 00000400:00000010:1.0:1788205181.788800:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000200:3.0:1788205181.788804:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205181.788807:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205181.788809:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205181.788811:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000010:1.0:1788205181.788813:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205181.788813:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.788814:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788818:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205181.788820:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205181.788821:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205181.788822:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205181.788823:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00000200:1.0:1788205181.788824:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543521024, portal 25 00000020:00000001:3.0:1788205181.788825:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205181.788829:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205181.788830:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1788205181.788832:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205181.788834:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000001:3.0:1788205181.788835:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.788840:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.788843:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788205181.788845:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205181.788847:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205181.788855:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205181.788860:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788205181.788861:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205181.788864:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1788205181.788867:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205181.788869:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1788205181.788872:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 02000000:00000001:3.0:1788205181.788873:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788877:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000200:1.0:1788205181.788879:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543521024, offset 0 00000008:00100000:3.0:1788205181.788880:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205181.788885:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788887:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205181.788892:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000010:3.0:1788205181.788893:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788205181.788897:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205181.788900:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205181.788901:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.788903:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1788205181.788905:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205181.788905:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205181.788909:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205181.788916:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000400:00000200:1.0:1788205181.788917:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788205181.788921:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205181.788926:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205181.788932:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205181.788937:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00100000:3.0:1788205181.788940:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205181.788942:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205181.788952:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471419:471419:256:4294967295] 192.168.201.118@tcp LPNI seq info [471278:471278:8:4294967295] 00000100:00000001:3.0:1788205181.788955:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205181.788960:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205181.788961:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.788973:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205181.788979:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.788986:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205181.788994:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205181.788998:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205181.789004:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205181.789008:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205181.789012:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.789015:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205181.789022:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205181.789027:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205181.789035:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205181.789038:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205181.789044:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205181.789048:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789058:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.789062:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205181.789073:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789074:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.789078:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205181.789087:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.789087:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205181.789117:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205181.789121:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205181.789123:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205181.789125:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205181.789128:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205181.789131:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26868 00000400:00000200:3.0:1788205181.789136:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205181.789139:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205181.789141:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205181.789143:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.789145:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.789147:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.789149:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205181.789152:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205181.789153:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205181.789156:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205181.789158:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205181.789160:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205181.789163:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205181.789166:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.789169:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205181.789171:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205181.789173:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205181.789175:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205181.789179:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205181.789181:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205181.789183:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205181.789186:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205181.789188:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.789190:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205181.789192:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205181.789195:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.789196:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205181.789201:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205181.789204:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205181.789207:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205181.789208:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205181.789210:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205181.789212:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205181.789216:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205181.789221:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000010:3.0:1788205181.789221:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:1.0:1788205181.789226:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321541894 00000100:00000001:3.0:1788205181.789227:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205181.789236:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647d80. 00000100:00100000:3.0:1788205181.789236:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205181.789246:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205181.789247:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205181.789249:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205181.789255:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789258:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321541894 00000100:00000010:1.0:1788205181.789264:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16474e0. 00000100:00000001:3.0:1788205181.789268:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.789270:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000040:3.0:1788205181.789275:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26878) 00000100:00000001:1.0:1788205181.789276:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205181.789282:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.789285:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205181.789292:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205181/real 1788205181] req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205181.789304:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205181.789306:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205181.789309:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205181.789313:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205181.789315:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205181.789318:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205181.789322:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205181.789332:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789339:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205181.789340:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789344:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.789346:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205181.789350:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789357:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205181.789362:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789370:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205181.789373:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789375:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.789376:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789379:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205181.789381:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205181.789384:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205181.789388:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205181.789391:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205181.789396:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205181.789404:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543521024:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205181.789411:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789414:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.789417:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543521024/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789424:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205181.789427:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205181.789430:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205181.789433:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205181.789435:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205181.789437:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205181.789439:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205181.789443:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:1.0:1788205181.789447:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205181.789449:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205181.789451:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205181.789455:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205181.789456:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789458:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789469:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205181.789477:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.789480:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205181.789482:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205181.789483:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205181.789487:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789494:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205181.789495:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.789497:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205181.789500:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205181.789502:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205181.789505:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521152:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205181.789508:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205181.789511:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205181.789512:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.789515:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205181.789518:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205181.789520:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205181.789524:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205181.789527:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205181.789531:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205181.789534:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543521152, portal 10 00000100:00000001:1.0:1788205181.789536:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205181.789537:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205181.789544:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789551:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205181.789553:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205181.789555:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205181.789558:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543521152, offset 0 00000400:00000200:1.0:1788205181.789562:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205181.789567:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.789571:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205181.789574:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205181.789577:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205181.789579:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205181.789581:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205181.789585:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471420:471420:256:4294967295] 192.168.201.118@tcp LPNI seq info [471279:471279:8:4294967295] 00000400:00000200:1.0:1788205181.789588:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.789594:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205181.789598:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.789601:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205181.789605:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205181.789608:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205181.789611:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205181.789615:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205181.789618:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.789619:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205181.789623:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205181.789626:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205181.789630:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205181.789631:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205181.789634:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205181.789646:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789655:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.789658:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789665:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.789666:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205181.789774:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205181.789779:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205181.789783:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205181.789786:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321540627 00000100:00000001:1.0:1788205181.789789:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321541894 00000100:00000001:1.0:1788205181.789791:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321540627 00000100:00000001:1.0:1788205181.789794:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321541894 00000100:00000001:1.0:1788205181.789796:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205181.789798:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.789801:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205181.789805:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205181/real 1788205181] req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205181.789814:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205181.789816:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205181.789818:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205181.789820:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205181.789822:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205181.789824:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205181.789828:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205181.789837:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789843:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205181.789844:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789846:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.789848:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205181.789851:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789858:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205181.789862:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789868:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205181.789870:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789872:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.789873:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.789875:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205181.789878:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205181.789880:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205181.789884:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205181.789886:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205181.789890:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205181.789898:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521152:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205181.789903:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205181.789906:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789918:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205181.789919:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.789922:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205181.789924:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205181.789926:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205181.789929:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521280:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205181.789932:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205181.789934:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205181.789936:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.789938:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205181.789941:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205181.789943:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205181.789947:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205181.789950:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205181.789954:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205181.789957:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543521280, portal 10 00000100:00000001:1.0:1788205181.789959:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205181.789960:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205181.789966:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.789973:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205181.789975:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205181.789977:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205181.789980:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543521280, offset 0 00000400:00000200:1.0:1788205181.789984:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205181.789988:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.789992:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205181.789994:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205181.789997:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205181.790000:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205181.790002:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205181.790006:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471421:471421:256:4294967295] 192.168.201.118@tcp LPNI seq info [471280:471280:8:4294967295] 00000400:00000200:1.0:1788205181.790010:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.790016:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205181.790020:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.790023:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205181.790027:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205181.790030:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205181.790034:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205181.790037:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205181.790041:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.790043:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205181.790045:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205181.790049:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205181.790052:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205181.790053:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205181.790056:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205181.790059:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790067:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.790070:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790076:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.790078:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205181.790081:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205181.790084:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205181.790087:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205181.790089:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321540628 00000100:00000001:1.0:1788205181.790113:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321541894 00000100:00000001:1.0:1788205181.790115:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321540629 00000100:00000001:1.0:1788205181.790117:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321541894 00000100:00000001:1.0:1788205181.790119:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205181.790122:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.790124:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205181.790128:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205181/real 1788205181] req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205181.790136:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205181.790137:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205181.790138:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205181.790141:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205181.790143:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205181.790145:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205181.790148:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205181.790156:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790162:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205181.790164:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.790165:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.790166:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205181.790170:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790176:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205181.790180:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790188:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205181.790190:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.790192:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.790193:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.790195:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205181.790197:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205181.790200:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205181.790204:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205181.790206:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205181.790209:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205181.790217:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521280:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205181.790220:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205181.790223:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790229:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205181.790231:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.790233:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205181.790235:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205181.790237:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205181.790240:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521408:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205181.790243:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205181.790245:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205181.790246:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.790248:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205181.790252:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205181.790254:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205181.790257:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205181.790260:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205181.790263:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205181.790265:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543521408, portal 4 00000100:00000001:1.0:1788205181.790267:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205181.790268:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205181.790273:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790280:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205181.790282:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205181.790284:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205181.790287:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543521408, offset 0 00000400:00000200:1.0:1788205181.790290:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205181.790294:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.790298:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205181.790301:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205181.790303:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205181.790305:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205181.790308:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205181.790311:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471422:471422:256:4294967295] 192.168.201.118@tcp LPNI seq info [471281:471281:8:4294967295] 00000400:00000200:1.0:1788205181.790314:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.790320:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205181.790324:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.790327:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205181.790331:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205181.790334:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205181.790338:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205181.790340:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205181.790344:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.790346:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205181.790349:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205181.790353:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205181.790356:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205181.790358:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205181.790360:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205181.790364:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790372:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.790375:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790381:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.790383:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205181.790386:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205181.790389:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205181.790393:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205181.790395:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321540638 00000100:00000001:1.0:1788205181.790397:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321541894 00000100:00000001:1.0:1788205181.790399:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321540638 00000100:00000001:1.0:1788205181.790402:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321541894 00000100:00000001:1.0:1788205181.790404:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205181.790406:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.790407:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205181.790411:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205181/real 1788205181] req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205181.790419:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205181.790421:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205181.790422:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205181.790425:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205181.790426:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205181.790428:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205181.790432:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205181.790440:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790446:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205181.790448:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.790450:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.790451:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205181.790454:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790461:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205181.790465:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790473:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205181.790475:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.790477:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.790478:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.790480:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205181.790482:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205181.790485:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205181.790488:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205181.790491:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205181.790495:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205181.790503:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521408:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205181.790506:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205181.790510:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790516:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205181.790518:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.790520:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205181.790522:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205181.790524:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205181.790527:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521536:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205181.790530:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205181.790532:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205181.790534:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205181.790535:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205181.790539:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205181.790541:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205181.790544:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205181.790546:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205181.790549:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205181.790552:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543521536, portal 4 00000100:00000001:1.0:1788205181.790554:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205181.790555:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205181.790560:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790567:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205181.790570:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205181.790572:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205181.790574:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543521536, offset 0 00000400:00000200:1.0:1788205181.790578:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205181.790581:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.790584:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205181.790587:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205181.790590:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205181.790591:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205181.790593:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205181.790596:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471423:471423:256:4294967295] 192.168.201.118@tcp LPNI seq info [471282:471282:8:4294967295] 00000400:00000200:1.0:1788205181.790600:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205181.790606:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205181.790611:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.790614:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205181.790618:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205181.790620:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205181.790624:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205181.790626:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205181.790629:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205181.790631:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205181.790633:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205181.790648:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205181.790651:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205181.790653:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205181.790657:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205181.790660:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.790668:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.791007:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205206 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.791016:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.791018:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205181.791023:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205181.791026:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205181.791030:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205181.791032:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321540661 00000100:00000001:1.0:1788205181.791034:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321541894 00000100:00000001:1.0:1788205181.791036:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321540661 00000100:00000001:1.0:1788205181.791038:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321541894 00000100:00000001:1.0:1788205181.791041:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205181.791043:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.791045:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205181.791049:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205181/real 1788205181] req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205181.791057:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205181.791058:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205181.791060:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205181.791062:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205181.791064:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205181.791066:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205181.791069:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205181.791077:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.791112:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205181.791115:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791117:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.791118:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205181.791122:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.791129:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205181.791132:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.791139:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205181.791141:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791143:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205181.791144:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791146:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205181.791148:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205181.791151:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205181.791154:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205181.791156:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205181.791160:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205181.791168:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521536:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205181.791172:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791175:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.791179:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543521152/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.791186:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205181.791187:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205181.791190:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205181.791193:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205181.791195:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205181.791196:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205181.791198:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205181.791201:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:1.0:1788205181.791204:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205181.791205:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205181.791212:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205181.791216:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205181.791217:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791219:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.791223:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543521280/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.791229:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205181.791230:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205181.791232:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205181.791234:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205181.791235:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205181.791236:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205181.791238:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205181.791240:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788205181.791242:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205181.791243:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205181.791245:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205181.791248:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205181.791249:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791250:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.791254:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543521408/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.791260:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205181.791261:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205181.791262:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205181.791264:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205181.791265:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205181.791267:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205181.791269:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205181.791270:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205181.791273:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205181.791274:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205181.791275:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205181.791277:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205181.791278:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791280:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205181.791283:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543521536/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205206 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205181.791288:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205181.791289:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205181.791291:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205181.791293:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205181.791294:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205181.791295:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205181.791297:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205181.791299:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205181.791301:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205181.791302:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205181.791304:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205181.791306:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205181.791307:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791309:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205181.791316:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205181.791318:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205181.791321:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205181.791322:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205182.620227:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205182.620228:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205182.620239:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205182.620239:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205182.620243:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205182.620243:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205182.620252:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205182.620252:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205182.620259:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205182.620259:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205182.620265:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205182.620265:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205182.620271:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205182.620271:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205182.620275:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205182.620276:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205182.620281:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205182.620282:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205182.620712:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205182.620716:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205182.620720:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205182.620720:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205182.620723:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205182.620724:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205182.620726:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205182.620727:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205182.620818:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205182.620821:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205182.620823:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205183.132230:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321541894, at 4321541952 00000100:00000010:3.0:1788205183.132253:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16474e0. 00000100:00000040:2.0:1788205183.132698:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321541894, at 4321541952 00000100:00000010:2.0:1788205183.132707:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647d80. 00000100:00000001:3.0:1788205183.644195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205183.644203:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205183.644205:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205183.644210:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205183.644214:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205183.644217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205183.644220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205183.644223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205183.644225:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205183.644672:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205183.644676:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205183.644678:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205183.644681:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205183.644683:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205183.644685:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205183.644687:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205183.644690:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205183.644692:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205183.646168:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205183.646175:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205183.646182:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205183.646184:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205183.646188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205183.646201:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205183.646206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205183.646208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205183.646237:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205183.646241:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205183.646245:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205184.668206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205184.668210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205184.668215:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205184.668215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205184.668216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205184.668219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205184.668224:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205184.668226:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205184.668226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205184.668228:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668232:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205184.668232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788205184.668233:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205184.668237:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205184.668238:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205184.668239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205184.668242:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205184.668245:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205184.668245:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205184.668247:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788205184.668250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205184.668251:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205184.668252:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205184.668256:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205184.668257:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205184.668257:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205184.668261:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668264:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205184.668272:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205184.668274:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205184.668275:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668277:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205184.668280:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668282:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668284:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205184.668293:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205184.668294:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205184.668296:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668298:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205184.668299:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668301:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205184.668303:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205184.668365:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205184.668370:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205184.668374:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205185.692187:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205185.692187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205185.692191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205185.692195:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.692195:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205185.692195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205185.692197:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205185.692199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205185.692202:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205185.692202:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205185.692204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205185.692205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.692206:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205185.692209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205185.692210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205185.692211:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205185.692213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.692214:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205185.692214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205185.692216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205185.692218:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205185.692219:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205185.692220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205185.692221:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205185.692223:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205185.692227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205185.692350:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205185.692354:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205185.692357:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205186.716268:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205186.716271:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205186.716272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205186.716276:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205186.716278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.716281:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205186.716282:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205186.716287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205186.716288:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205186.716292:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205186.716294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205186.716297:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205186.716299:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205186.716303:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205186.716305:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205186.716308:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205186.716309:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205186.716312:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205186.716315:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205186.716316:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205186.716318:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205186.716322:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205186.716326:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205186.716326:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205186.716327:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205186.716331:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205186.716369:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205186.716372:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205186.716375:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205186.908298:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205186.908308:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205186.908310:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205186.908313:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205186.908317:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205186.908322:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26873 00000400:00000200:2.0:1788205186.908337:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205186.908344:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205186.908347:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205186.908350:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908354:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205186.908356:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908359:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205186.908364:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205186.908366:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205186.908369:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205186.908372:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205186.908374:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205186.908376:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205186.908379:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908381:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205186.908383:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908385:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205186.908387:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205186.908391:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908393:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205186.908396:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205186.908399:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205186.908412:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.908415:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205186.908418:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205186.908430:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788205186.908434:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205186.908437:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205186.908439:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.908443:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205186.908446:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205186.908451:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205186.908458:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788205186.908464:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205186.908474:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205186.908484:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205186.908489:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205186.908494:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205186.908521:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205186.908534:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205186.908537:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205186.908539:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205186.908540:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205186.908543:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205186.908546:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26873 00000400:00000200:2.0:1788205186.908563:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205186.908567:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205186.908570:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205186.908573:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908576:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205186.908577:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908580:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205186.908582:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205186.908583:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788205186.908585:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205186.908587:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788205186.908589:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788205186.908592:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788205186.908595:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908597:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205186.908599:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908600:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205186.908603:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788205186.908606:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908608:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205186.908610:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205186.908617:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205186.908619:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.908621:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205186.908624:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205186.908627:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.908629:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205186.908632:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788205186.908635:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205186.908637:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205186.908638:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.908640:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205186.908641:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205186.908643:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205186.908646:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788205186.908650:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205186.908655:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205186.908663:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205186.908665:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.908675:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1788205186.908685:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.908695:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205186.908697:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.908699:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788205186.908701:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205186.908708:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:1.0:1788205186.908709:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205186.908710:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205186.908713:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205186.908718:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205186.908724:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205186.908727:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.908730:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205186.908734:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788205186.908734:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26873 02000000:00000001:1.0:1788205186.908738:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205186.908743:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543521664:192.168.201.118@tcp:250:kworker.0 00000400:00000200:2.0:1788205186.908743:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205186.908749:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:2.0:1788205186.908752:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 02000000:00000001:1.0:1788205186.908753:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205186.908756:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205186.908756:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788205186.908760:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1788205186.908763:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000010:1.0:1788205186.908765:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 00000100:00000001:2.0:1788205186.908766:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788205186.908768:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205186.908772:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908775:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:1.0:1788205186.908776:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:2.0:1788205186.908780:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000010:1.0:1788205186.908781:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000020:00000040:2.0:1788205186.908783:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788205186.908786:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000010:1.0:1788205186.908788:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000020:00000040:2.0:1788205186.908789:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788205186.908793:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788205186.908796:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000200:1.0:1788205186.908798:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543521664, portal 25 00000100:00000040:2.0:1788205186.908801:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205186.908803:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205186.908805:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205186.908806:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913984 : -109424377637632 : ffff9c7aa6d75500) 00000100:00000040:2.0:1788205186.908809:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908813:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788205186.908817:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205186.908818:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000001:1.0:1788205186.908828:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205186.908831:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788205186.908835:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205186.908839:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205186.908841:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1788205186.908844:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 02000000:00000001:2.0:1788205186.908844:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205186.908850:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205186.908852:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543521664, offset 0 00000100:00000001:2.0:1788205186.908852:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205186.908855:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205186.908860:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.908863:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.908864:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000010:2.0:1788205186.908869:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788205186.908873:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000200:1.0:1788205186.908875:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.908881:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205186.908888:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1788205186.908888:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205186.908891:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.908895:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:2.0:1788205186.908896:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205186.908899:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205186.908901:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205186.908905:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788205186.908909:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.908913:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00100000:2.0:1788205186.908916:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205186.908920:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205186.908929:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471424:471424:256:4294967295] 192.168.201.118@tcp LPNI seq info [471283:471283:8:4294967295] 00000100:00000001:2.0:1788205186.908930:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205186.908934:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.908939:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:2.0:1788205186.908943:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205186.908958:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:2.0:1788205186.908959:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205186.908965:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1788205186.908969:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788205186.908971:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205186.908974:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:1.0:1788205186.908976:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00080000:2.0:1788205186.908976:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205186.908982:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205186.908987:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26873 00000800:00000200:1.0:1788205186.908988:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205186.908994:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000400:00000200:2.0:1788205186.908994:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205186.909000:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205186.909004:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205186.909006:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.909008:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205186.909013:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.1:1788205186.909014:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788205186.909017:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.1:1788205186.909020:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1788205186.909021:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205186.909024:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788205186.909027:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000100:1.0:1788205186.909029:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000040:2.0:1788205186.909030:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000800:00000010:1.0:1788205186.909035:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000020:00000001:2.0:1788205186.909035:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205186.909041:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788205186.909045:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205186.909046:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205186.909048:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205186.909054:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205186.909057:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788205186.909058:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205186.909064:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000200:1.0:1788205186.909067:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00080000:2.0:1788205186.909068:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:1.0:1788205186.909072:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:2.0:1788205186.909073:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205186.909077:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205186.909079:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1788205186.909081:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205186.909085:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205186.909086:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205186.909088:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205186.909124:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788205186.909126:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.909127:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205186.909132:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788205186.909135:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205186.909139:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205186.909142:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909144:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788205186.909145:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205186.909149:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1788205186.909152:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205186.909152:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205186.909159:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:1.0:1788205186.909165:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.909166:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909170:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1788205186.909175:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205186.909177:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.909186:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788205186.909189:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205186.909191:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321547014 00000100:00000001:2.0:1788205186.909193:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205186.909199:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647a80. 00000100:00000040:2.0:1788205186.909200:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205186.909208:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909209:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00080000:2.0:1788205186.909210:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205186.909214:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205186.909215:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205186.909216:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321547014 00000100:00080000:2.0:1788205186.909217:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000010:1.0:1788205186.909222:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647900. 00000100:00000001:2.0:1788205186.909223:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205186.909227:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00080000:2.0:1788205186.909229:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26873 00000100:00000001:1.0:1788205186.909231:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205186.909234:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909236:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1788205186.909237:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:1.0:1788205186.909241:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205186/real 1788205186] req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205186.909244:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205186.909249:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205186.909255:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000001:1.0:1788205186.909256:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205186.909258:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788205186.909261:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1788205186.909262:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205186.909266:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205186.909268:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000040:2.0:1788205186.909268:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205186.909272:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1788205186.909273:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000200:1.0:1788205186.909277:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205186.909278:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205186.909280:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788205186.909282:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205186.909285:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788205186.909288:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788205186.909290:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788205186.909293:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.909294:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205186.909295:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205186.909298:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000200:1.0:1788205186.909299:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205186.909300:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788205186.909303:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205186.909307:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205186.909310:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205186.909315:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205186.909316:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788205186.909318:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909320:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205186.909320:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205186.909323:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205186.909325:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:2.0:1788205186.909326:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909328:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205186.909329:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205186.909335:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000020:00000040:2.0:1788205186.909338:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00100000:1.0:1788205186.909339:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205186.909340:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205186.909342:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205186.909345:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205186.909348:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205186.909350:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909353:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1788205186.909354:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000040:1.0:1788205186.909358:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205186.909360:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205186.909367:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205186.909368:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205186.909373:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.909377:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205186.909378:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205186.909379:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205186.909381:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909383:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205186.909386:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1788205186.909386:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205186.909391:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205186.909398:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:2.0:1788205186.909399:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909400:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1788205186.909407:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26883) 00000100:00000040:1.0:1788205186.909417:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205186.909425:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543521664:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205186.909430:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.909433:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.909436:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75500 x1875055543521664/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.909442:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205186.909445:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205186.909447:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205186.909450:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205186.909452:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205186.909453:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205186.909455:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205186.909457:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d000. 02000000:00000001:1.0:1788205186.909459:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205186.909461:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205186.909462:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75500. 00000100:00000001:1.0:1788205186.909464:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205186.909465:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.909466:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.909472:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205186.909473:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.909475:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205186.909476:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205186.909477:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205186.909479:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.909488:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205186.909489:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.909490:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205186.909491:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205186.909492:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205186.909494:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521792:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205186.909496:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205186.909497:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205186.909498:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.909499:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205186.909501:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205186.909502:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.909505:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205186.909507:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205186.909509:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205186.909512:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543521792, portal 10 00000100:00000001:1.0:1788205186.909514:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205186.909515:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331909504 : -109424377642112 : ffff9c7aa6d74380) 00000100:00000040:1.0:1788205186.909520:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.909526:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205186.909528:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205186.909530:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205186.909532:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543521792, offset 0 00000400:00000200:1.0:1788205186.909991:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205186.910002:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.910005:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205186.910007:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205186.910009:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205186.910011:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205186.910012:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205186.910015:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471425:471425:256:4294967295] 192.168.201.118@tcp LPNI seq info [471284:471284:8:4294967295] 00000400:00000200:1.0:1788205186.910018:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.910022:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205186.910025:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205186.910029:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205186.910033:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205186.910036:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205186.910040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205186.910042:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205186.910046:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205186.910048:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205186.910051:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205186.910055:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205186.910058:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205186.910059:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205186.910061:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205186.910064:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910070:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.910072:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910076:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910077:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205186.910079:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205186.910082:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205186.910084:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205186.910085:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321545747 00000100:00000001:1.0:1788205186.910088:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321547014 00000100:00000001:1.0:1788205186.910089:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321545747 00000100:00000001:1.0:1788205186.910104:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321547014 00000100:00000001:1.0:1788205186.910106:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205186.910108:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910109:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205186.910111:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205186/real 1788205186] req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205186.910117:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205186.910118:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205186.910119:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205186.910120:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205186.910122:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205186.910123:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205186.910125:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205186.910130:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910134:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205186.910134:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910136:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205186.910136:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205186.910138:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910142:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205186.910144:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910147:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205186.910148:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910149:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205186.910150:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910151:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205186.910152:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205186.910154:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205186.910156:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205186.910157:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205186.910160:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205186.910164:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521792:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205186.910166:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205186.910168:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910171:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205186.910172:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.910173:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205186.910174:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205186.910175:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205186.910177:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521920:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205186.910179:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205186.910180:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205186.910181:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.910183:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205186.910185:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205186.910186:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.910188:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205186.910190:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205186.910191:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205186.910193:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543521920, portal 10 00000100:00000001:1.0:1788205186.910194:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205186.910195:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331911296 : -109424377640320 : ffff9c7aa6d74a80) 00000100:00000040:1.0:1788205186.910199:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910203:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205186.910204:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205186.910206:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205186.910207:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543521920, offset 0 00000400:00000200:1.0:1788205186.910209:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205186.910212:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.910214:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205186.910216:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205186.910217:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205186.910219:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205186.910220:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205186.910222:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471426:471426:256:4294967295] 192.168.201.118@tcp LPNI seq info [471285:471285:8:4294967295] 00000400:00000200:1.0:1788205186.910225:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.910229:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205186.910232:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205186.910234:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205186.910238:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205186.910241:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205186.910244:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205186.910247:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205186.910250:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205186.910252:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205186.910254:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205186.910257:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205186.910260:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205186.910262:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205186.910264:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205186.910268:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910275:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.910278:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910284:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910286:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205186.910289:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205186.910291:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205186.910295:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205186.910297:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321545749 00000100:00000001:1.0:1788205186.910299:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321547014 00000100:00000001:1.0:1788205186.910301:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321545749 00000100:00000001:1.0:1788205186.910302:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321547014 00000100:00000001:1.0:1788205186.910304:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205186.910306:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910308:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205186.910311:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205186/real 1788205186] req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205186.910318:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205186.910320:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205186.910321:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205186.910324:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205186.910325:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205186.910327:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205186.910330:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205186.910338:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910342:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205186.910343:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910345:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205186.910346:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205186.910348:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910352:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205186.910354:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910358:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205186.910360:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910361:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205186.910361:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910362:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205186.910364:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205186.910365:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205186.910367:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205186.910368:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205186.910370:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205186.910375:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543521920:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205186.910377:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205186.910379:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910382:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205186.910383:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.910385:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205186.910386:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205186.910387:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205186.910389:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522048:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205186.910390:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205186.910392:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205186.910393:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.910394:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205186.910396:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205186.910398:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.910400:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205186.910402:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205186.910405:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205186.910407:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543522048, portal 4 00000100:00000001:1.0:1788205186.910409:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205186.910410:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:1.0:1788205186.910415:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910421:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205186.910423:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205186.910425:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205186.910428:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543522048, offset 0 00000400:00000200:1.0:1788205186.910432:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205186.910435:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.910439:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205186.910441:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205186.910444:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205186.910446:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205186.910448:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205186.910450:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471427:471427:256:4294967295] 192.168.201.118@tcp LPNI seq info [471286:471286:8:4294967295] 00000400:00000200:1.0:1788205186.910453:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.910457:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205186.910460:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205186.910462:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205186.910464:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205186.910466:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205186.910469:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205186.910471:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205186.910473:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205186.910474:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205186.910476:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205186.910478:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205186.910479:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205186.910481:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205186.910483:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205186.910485:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910490:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.910492:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910496:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910497:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205186.910498:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205186.910500:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205186.910502:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205186.910503:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321545758 00000100:00000001:1.0:1788205186.910505:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321547014 00000100:00000001:1.0:1788205186.910507:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321545758 00000100:00000001:1.0:1788205186.910508:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321547014 00000100:00000001:1.0:1788205186.910509:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205186.910511:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910512:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205186.910516:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205186/real 1788205186] req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205186.910523:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205186.910524:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205186.910525:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205186.910528:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205186.910529:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205186.910531:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205186.910534:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205186.910542:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910558:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205186.910560:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910562:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205186.910563:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205186.910567:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910573:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205186.910577:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910581:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205186.910583:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910584:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205186.910584:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910586:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205186.910587:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205186.910588:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205186.910590:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205186.910591:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205186.910593:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205186.910598:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522048:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205186.910600:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205186.910602:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910605:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205186.910606:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.910608:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205186.910609:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205186.910610:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205186.910612:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522176:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205186.910614:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205186.910615:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205186.910616:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205186.910617:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205186.910619:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205186.910620:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205186.910622:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205186.910624:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205186.910626:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205186.910628:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543522176, portal 4 00000100:00000001:1.0:1788205186.910629:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205186.910630:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:1.0:1788205186.910634:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910637:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205186.910639:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205186.910640:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205186.910641:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543522176, offset 0 00000400:00000200:1.0:1788205186.910644:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205186.910646:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.910648:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205186.910650:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205186.910651:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205186.910653:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205186.910654:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205186.910657:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471428:471428:256:4294967295] 192.168.201.118@tcp LPNI seq info [471287:471287:8:4294967295] 00000400:00000200:1.0:1788205186.910661:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205186.910667:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205186.910671:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205186.910674:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205186.910677:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205186.910680:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205186.910684:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205186.910686:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205186.910690:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205186.910692:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205186.910695:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205186.910698:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205186.910701:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205186.910702:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205186.910705:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205186.910707:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910712:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.910714:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205211 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910718:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910719:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205186.910721:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205186.910723:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205186.910725:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205186.910726:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321545781 00000100:00000001:1.0:1788205186.910728:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321547014 00000100:00000001:1.0:1788205186.910729:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321545781 00000100:00000001:1.0:1788205186.910730:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321547014 00000100:00000001:1.0:1788205186.910731:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205186.910733:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910734:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205186.910736:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205186/real 1788205186] req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205186.910741:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205186.910741:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205186.910743:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205186.910744:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205186.910745:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205186.910746:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205186.910747:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205186.910752:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910755:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205186.910756:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910757:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205186.910757:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205186.910759:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910763:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205186.910765:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910768:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205186.910769:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910770:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205186.910771:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910772:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205186.910773:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205186.910775:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205186.910777:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205186.910778:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205186.910780:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205186.910785:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522176:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205186.910787:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910788:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.910789:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74380 x1875055543521792/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910793:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205186.910794:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205186.910795:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205186.910797:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205186.910797:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205186.910798:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205186.910800:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205186.910801:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:1.0:1788205186.910803:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205186.910804:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205186.910805:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74380. 00000100:00000001:1.0:1788205186.910806:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205186.910807:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910808:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.910809:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74a80 x1875055543521920/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910813:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205186.910814:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205186.910815:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205186.910816:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205186.910817:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205186.910818:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205186.910819:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205186.910820:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:1.0:1788205186.910822:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205186.910823:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205186.910824:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74a80. 00000100:00000001:1.0:1788205186.910826:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205186.910827:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910829:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.910831:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77b80 x1875055543522048/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910847:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205186.910848:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205186.910850:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205186.910853:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205186.910854:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205186.910855:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205186.910857:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205186.910858:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788205186.910861:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205186.910862:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205186.910863:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77b80. 00000100:00000001:1.0:1788205186.910866:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205186.910867:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910868:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205186.910871:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543522176/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205211 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205186.910875:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205186.910876:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205186.910877:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205186.910878:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205186.910879:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205186.910879:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205186.910880:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205186.910881:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067ec00. 02000000:00000001:1.0:1788205186.910882:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205186.910883:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205186.910883:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788205186.910884:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205186.910885:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910887:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205186.910890:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205186.910891:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205186.910892:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205186.910893:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205187.740216:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205187.740217:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205187.740229:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205187.740229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205187.740232:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205187.740233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205187.740241:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205187.740242:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205187.740248:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205187.740249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205187.740255:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.740255:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.740260:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205187.740261:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205187.740265:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205187.740266:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205187.740269:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205187.740271:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205187.740630:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205187.740635:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205187.740640:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205187.740642:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205187.740646:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205187.740647:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205187.740651:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205187.740652:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205187.740709:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205187.740714:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205187.740718:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205188.252398:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321547014, at 4321547072 00000100:00000010:2.0:1788205188.252430:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647a80. 00000100:00000040:3.0:1788205188.252515:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321547014, at 4321547072 00000100:00000010:3.0:1788205188.252522:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647900. 00010000:00000001:0.0:1788205188.764283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205188.764294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205188.764302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205188.764309:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205188.764313:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205188.764314:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205188.764318:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205188.764319:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205188.764534:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205188.764539:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205188.764543:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205188.764569:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205188.764576:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205188.764577:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205188.764581:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205188.764582:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205188.764585:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205188.764587:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205188.764588:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205188.764593:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205188.764594:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205188.764596:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205188.764598:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205188.764599:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205188.764604:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205188.764604:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205188.764609:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205188.764609:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205188.764615:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205189.788200:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205189.788200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205189.788207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205189.788209:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205189.788210:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205189.788213:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205189.788214:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205189.788215:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205189.788220:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:3.0:1788205189.788221:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205189.788222:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205189.788225:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205189.788226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205189.788228:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205189.788230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205189.788231:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205189.788233:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205189.788235:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205189.788235:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205189.788239:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205189.788240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205189.788241:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205189.788243:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205189.788247:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205189.788248:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205189.788254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205189.788254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205189.788258:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205189.788263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205189.788265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205189.788270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205189.788272:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205189.788298:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205189.788301:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205189.788303:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205190.812221:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205190.812221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205190.812235:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205190.812235:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205190.812241:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205190.812241:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205190.812250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205190.812254:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205190.812257:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205190.812258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205190.812261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205190.812265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.812267:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205190.812267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205190.812270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205190.812277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.812278:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205190.812279:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205190.812289:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205190.812290:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205190.812291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205190.812296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205190.812297:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205190.812298:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205190.812309:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205190.812320:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205190.812364:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205190.812369:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205190.812372:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205191.836160:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205191.836164:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205191.836167:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205191.836168:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205191.836169:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205191.836176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205191.836177:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205191.836180:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205191.836181:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205191.836184:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205191.836185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205191.836186:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205191.836190:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205191.836193:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205191.836194:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205191.836196:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205191.836198:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205191.836248:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205191.836251:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205191.836254:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205191.836293:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205191.836299:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205191.836300:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205191.836304:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205191.836306:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205191.836309:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205191.836311:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205191.836314:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205191.836316:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205192.028291:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205192.028305:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205192.028317:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205192.028322:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205192.028327:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205192.028337:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26878 00000400:00000200:3.0:1788205192.028351:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205192.028359:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205192.028363:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205192.028367:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028371:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205192.028374:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028377:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205192.028382:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205192.028385:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205192.028388:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205192.028392:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205192.028395:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205192.028398:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205192.028402:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028404:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205192.028406:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028408:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205192.028411:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205192.028414:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028417:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205192.028421:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205192.028426:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205192.028429:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.028432:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205192.028435:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205192.028491:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205192.028496:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205192.028501:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205192.028503:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.028507:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205192.028510:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205192.028515:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205192.028528:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788205192.028537:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205192.028549:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205192.028561:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205192.028567:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205192.028572:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205192.028602:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205192.028606:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205192.028611:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205192.028612:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205192.028613:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205192.028617:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205192.028620:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26878 00000400:00000200:3.0:1788205192.028624:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205192.028627:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205192.028629:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205192.028632:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028634:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205192.028636:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028638:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205192.028649:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205192.028651:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205192.028653:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205192.028656:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205192.028658:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205192.028662:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205192.028666:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028669:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205192.028671:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028673:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205192.028676:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205192.028679:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028681:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205192.028683:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205192.028687:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788205192.028691:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205192.028695:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.028697:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205192.028701:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205192.028704:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.028708:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.028710:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000010:3.0:1788205192.028713:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205192.028716:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1788205192.028726:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000040:1.0:1788205192.028734:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205192.028734:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205192.028740:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.028747:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205192.028749:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205192.028751:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205192.028761:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:1.0:1788205192.028774:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205192.028774:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.028780:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.028787:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205192.028792:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788205192.028794:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788205192.028796:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205192.028808:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543522304:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205192.028816:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205192.028820:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.028828:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1788205192.028833:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205192.028837:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205192.028842:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.028851:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788205192.028856:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205192.028863:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:1.0:1788205192.028864:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00080000:3.0:1788205192.028871:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1788205192.028872:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.028875:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205192.028879:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205192.028883:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205192.028885:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205192.028890:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26878 00000400:00000010:1.0:1788205192.028894:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000200:3.0:1788205192.028900:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205192.028905:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000010:1.0:1788205192.028906:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00080000:3.0:1788205192.028910:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205192.028917:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000200:1.0:1788205192.028918:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543522304, portal 25 00000100:00000001:3.0:1788205192.028925:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205192.028927:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1788205192.028932:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205192.028933:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000001:3.0:1788205192.028940:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205192.028945:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000040:1.0:1788205192.028949:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205192.028949:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205192.028955:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205192.028958:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205192.028963:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205192.028969:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205192.028979:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205192.028981:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788205192.028984:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205192.028987:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.028989:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205192.028992:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205192.028994:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029001:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205192.029004:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205192.029006:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1788205192.029011:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 02000000:00000001:3.0:1788205192.029013:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1788205192.029020:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543522304, offset 0 02000000:00000001:3.0:1788205192.029027:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.029033:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029034:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205192.029040:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205192.029045:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.029049:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.029051:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:3.0:1788205192.029058:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205192.029063:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000200:1.0:1788205192.029065:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 02000000:00000001:3.0:1788205192.029068:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205192.029070:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.029072:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205192.029074:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205192.029076:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205192.029080:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000400:00000200:1.0:1788205192.029084:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788205192.029087:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.029136:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00100000:3.0:1788205192.029141:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205192.029147:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205192.029158:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029165:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205192.029170:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.029172:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471429:471429:256:4294967295] 192.168.201.118@tcp LPNI seq info [471288:471288:8:4294967295] 00000100:00000040:3.0:1788205192.029179:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205192.029190:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205192.029202:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205192.029206:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205192.029210:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205192.029212:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205192.029216:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205192.029225:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205192.029231:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:3.0:1788205192.029233:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26878 00000400:00000200:3.0:1788205192.029249:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:1.0:1788205192.029250:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205192.029260:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000010:1.0:1788205192.029262:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00080000:3.0:1788205192.029264:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205192.029272:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029280:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205192.029284:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029286:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205192.029289:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205192.029290:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000800:00000200:1.0:1788205192.029293:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:3.0:1788205192.029293:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205192.029299:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000800:00000200:1.0:1788205192.029303:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000020:00000001:3.0:1788205192.029305:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205192.029314:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:1.1:1788205192.029323:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:3.0:1788205192.029324:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029336:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1788205192.029338:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000100:00000040:3.0:1788205192.029342:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029348:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000100:1.0:1788205192.029353:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000040:3.0:1788205192.029355:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000800:00000010:1.0:1788205192.029359:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00080000:3.0:1788205192.029365:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029369:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205192.029371:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205192.029375:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205192.029379:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.029381:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205192.029382:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00100000:3.0:1788205192.029385:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205192.029393:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.029398:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788205192.029398:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205192.029409:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000400:00000200:1.0:1788205192.029412:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000020:00000040:3.0:1788205192.029415:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000400:00000010:1.0:1788205192.029417:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 02000000:00000001:3.0:1788205192.029421:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205192.029425:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.029427:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205192.029429:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205192.029431:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205192.029436:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:3.0:1788205192.029439:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205192.029445:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205192.029453:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205192.029455:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205192.029459:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205192.029481:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205192.029484:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205192.029487:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205192.029488:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205192.029490:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205192.029493:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205192.029495:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26878 00000400:00000200:3.0:1788205192.029499:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205192.029502:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205192.029504:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205192.029507:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029509:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205192.029510:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029512:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205192.029514:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205192.029515:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205192.029516:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205192.029518:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205192.029521:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205192.029523:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205192.029526:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029528:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205192.029530:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029531:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205192.029533:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205192.029536:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205192.029538:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205192.029539:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205192.029543:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205192.029545:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.029547:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205192.029549:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205192.029551:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.029553:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205192.029557:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205192.029560:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205192.029562:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205192.029564:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.029565:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205192.029567:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205192.029569:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205192.029572:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788205192.029576:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205192.029581:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205192.029587:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205192.029590:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205192.029593:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205192.029601:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205192.029606:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26888) 00000100:00000001:1.0:1788205192.029868:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205192.029876:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.029889:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.029893:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.029901:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.029903:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205192.029910:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.029915:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205192.029918:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321552134 00000100:00000010:1.0:1788205192.029925:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647660. 00000100:00000001:1.0:1788205192.029931:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205192.029938:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321552134 00000100:00000010:1.0:1788205192.029941:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647840. 00000100:00000001:1.0:1788205192.029944:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205192.029948:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205192.029950:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.029953:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205192.029957:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205192/real 1788205192] req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205192.029966:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205192.029968:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205192.029970:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205192.029973:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205192.029974:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205192.029977:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205192.030039:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205192.030050:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030061:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205192.030063:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030065:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.030067:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205192.030143:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030150:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205192.030154:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030161:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205192.030164:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030166:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.030167:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030170:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205192.030172:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205192.030175:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205192.030180:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205192.030183:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205192.030187:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205192.030195:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543522304:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205192.030199:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030201:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.030204:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543522304/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030211:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205192.030213:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205192.030217:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205192.030221:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205192.030224:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205192.030226:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205192.030228:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205192.030230:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205192.030234:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205192.030236:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205192.030238:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205192.030241:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205192.030242:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030244:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030252:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205192.030254:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.030256:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205192.030258:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205192.030260:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205192.030264:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030272:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205192.030273:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.030276:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205192.030286:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205192.030289:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205192.030293:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522432:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205192.030297:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205192.030300:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205192.030302:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.030304:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205192.030308:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205192.030310:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.030314:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205192.030317:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205192.030321:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205192.030325:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543522432, portal 10 00000100:00000001:1.0:1788205192.030328:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205192.030329:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205192.030335:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030342:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205192.030344:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205192.030346:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205192.030349:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543522432, offset 0 00000400:00000200:1.0:1788205192.030354:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205192.030359:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205192.030372:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205192.030375:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205192.030378:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205192.030381:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205192.030384:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205192.030387:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471430:471430:256:4294967295] 192.168.201.118@tcp LPNI seq info [471289:471289:8:4294967295] 00000400:00000200:1.0:1788205192.030390:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205192.030398:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205192.030402:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205192.030406:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205192.030411:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205192.030414:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205192.030418:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205192.030428:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205192.030432:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205192.030433:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205192.030436:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205192.030440:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205192.030443:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205192.030445:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205192.030447:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205192.030451:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030475:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.030478:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030484:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.030486:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205192.030489:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205192.030493:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205192.030496:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205192.030498:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321550868 00000100:00000001:1.0:1788205192.030501:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321552134 00000100:00000001:1.0:1788205192.030502:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321550868 00000100:00000001:1.0:1788205192.030504:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321552134 00000100:00000001:1.0:1788205192.030507:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205192.030508:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.030510:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205192.030514:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205192/real 1788205192] req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205192.030522:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205192.030523:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205192.030525:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205192.030527:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205192.030529:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205192.030531:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205192.030534:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205192.030542:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030548:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205192.030550:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030552:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.030553:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205192.030557:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030564:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205192.030567:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030574:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205192.030577:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030578:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.030580:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030582:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205192.030584:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205192.030587:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205192.030590:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205192.030592:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205192.030596:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205192.030603:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522432:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205192.030607:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205192.030611:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030617:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205192.030619:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.030621:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205192.030623:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205192.030625:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205192.030628:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522560:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205192.030631:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205192.030633:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205192.030635:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.030636:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205192.030640:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205192.030642:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.030645:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205192.030647:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205192.030650:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205192.030653:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543522560, portal 10 00000100:00000001:1.0:1788205192.030655:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205192.030656:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205192.030662:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030668:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205192.030670:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205192.030673:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205192.030675:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543522560, offset 0 00000400:00000200:1.0:1788205192.030679:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205192.030683:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205192.030687:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205192.030689:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205192.030692:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205192.030704:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205192.030706:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205192.030709:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471431:471431:256:4294967295] 192.168.201.118@tcp LPNI seq info [471290:471290:8:4294967295] 00000400:00000200:1.0:1788205192.030712:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205192.030718:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205192.030722:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205192.030725:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205192.030730:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205192.030732:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205192.030736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205192.030739:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205192.030742:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205192.030744:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205192.030747:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205192.030750:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205192.030753:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205192.030754:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205192.030756:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205192.030759:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030767:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.030769:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030775:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.030777:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205192.030780:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205192.030782:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205192.030786:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205192.030787:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321550869 00000100:00000001:1.0:1788205192.030790:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321552134 00000100:00000001:1.0:1788205192.030792:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321550869 00000100:00000001:1.0:1788205192.030794:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321552134 00000100:00000001:1.0:1788205192.030796:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205192.030798:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.030800:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205192.030804:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205192/real 1788205192] req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205192.030812:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205192.030814:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205192.030815:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205192.030817:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205192.030819:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205192.030821:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205192.030835:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205192.030847:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030854:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205192.030855:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030856:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.030858:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205192.030861:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030867:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205192.030871:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030878:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205192.030880:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030881:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.030882:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.030884:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205192.030887:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205192.030889:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205192.030892:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205192.030894:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205192.030897:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205192.030910:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522560:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205192.030914:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205192.030918:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030924:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205192.030926:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.030928:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205192.030930:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205192.030932:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205192.030934:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522688:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205192.030937:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205192.030940:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205192.030941:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.030943:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205192.030946:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205192.030948:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.030951:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205192.030954:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205192.030957:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205192.030960:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543522688, portal 4 00000100:00000001:1.0:1788205192.030962:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205192.030963:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205192.030968:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.030974:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205192.030976:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205192.030978:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205192.030980:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543522688, offset 0 00000400:00000200:1.0:1788205192.030993:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205192.030997:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205192.031001:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205192.031004:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205192.031007:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205192.031009:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205192.031011:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205192.031013:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471432:471432:256:4294967295] 192.168.201.118@tcp LPNI seq info [471291:471291:8:4294967295] 00000400:00000200:1.0:1788205192.031017:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205192.031021:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205192.031025:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205192.031028:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205192.031032:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205192.031034:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205192.031039:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205192.031042:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205192.031046:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205192.031048:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205192.031050:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205192.031053:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205192.031056:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205192.031058:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205192.031060:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205192.031064:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031079:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.031085:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031113:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.031114:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205192.031118:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205192.031120:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205192.031124:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205192.031126:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321550879 00000100:00000001:1.0:1788205192.031128:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321552134 00000100:00000001:1.0:1788205192.031130:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321550879 00000100:00000001:1.0:1788205192.031132:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321552134 00000100:00000001:1.0:1788205192.031134:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205192.031136:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.031138:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205192.031142:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205192/real 1788205192] req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205192.031150:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205192.031151:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205192.031152:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205192.031154:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205192.031156:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205192.031158:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205192.031161:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205192.031169:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031175:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205192.031176:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031178:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.031179:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205192.031182:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031189:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205192.031192:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031199:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205192.031202:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031203:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.031204:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031206:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205192.031207:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205192.031210:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205192.031213:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205192.031215:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205192.031218:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205192.031226:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522688:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205192.031229:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205192.031233:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031239:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205192.031240:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.031242:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205192.031244:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205192.031246:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205192.031249:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522816:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205192.031252:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205192.031254:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205192.031255:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205192.031257:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205192.031260:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205192.031262:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205192.031265:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205192.031267:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205192.031271:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205192.031273:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543522816, portal 4 00000100:00000001:1.0:1788205192.031275:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205192.031277:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205192.031282:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031287:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205192.031289:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205192.031291:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205192.031293:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543522816, offset 0 00000400:00000200:1.0:1788205192.031296:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205192.031300:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205192.031303:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205192.031306:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205192.031308:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205192.031309:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205192.031311:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205192.031314:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471433:471433:256:4294967295] 192.168.201.118@tcp LPNI seq info [471292:471292:8:4294967295] 00000400:00000200:1.0:1788205192.031317:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205192.031322:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205192.031326:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205192.031328:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205192.031332:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205192.031334:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1788205192.031338:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205192.031341:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1788205192.031344:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205192.031346:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205192.031349:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205192.031352:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205192.031354:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205192.031356:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205192.031358:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205192.031361:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031373:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.031375:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205217 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031381:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.031382:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205192.031384:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205192.031387:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205192.031389:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205192.031391:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321550902 00000100:00000001:1.0:1788205192.031393:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321552134 00000100:00000001:1.0:1788205192.031395:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321550902 00000100:00000001:1.0:1788205192.031396:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321552134 00000100:00000001:1.0:1788205192.031398:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205192.031399:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.031401:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205192.031403:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205192/real 1788205192] req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205192.031410:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205192.031411:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205192.031412:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205192.031414:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205192.031415:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205192.031417:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205192.031419:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205192.031426:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031431:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205192.031433:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031434:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.031435:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205192.031438:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031443:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205192.031446:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031452:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205192.031454:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031455:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205192.031456:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031457:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205192.031459:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205192.031474:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205192.031477:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205192.031479:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205192.031483:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205192.031490:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543522816:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205192.031493:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031495:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.031498:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543522432/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031505:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205192.031507:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205192.031509:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205192.031511:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205192.031513:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205192.031515:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205192.031516:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205192.031518:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205192.031521:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205192.031522:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205192.031523:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205192.031526:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205192.031527:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031529:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.031532:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543522560/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031539:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205192.031540:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205192.031541:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205192.031543:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205192.031544:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205192.031545:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205192.031547:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205192.031548:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205192.031551:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205192.031552:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205192.031553:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205192.031556:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205192.031557:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031558:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.031561:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543522688/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031566:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205192.031568:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205192.031569:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205192.031571:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205192.031572:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205192.031573:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205192.031575:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205192.031576:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205192.031578:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205192.031580:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205192.031581:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205192.031583:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205192.031584:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031586:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205192.031588:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543522816/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205217 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205192.031594:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205192.031595:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205192.031596:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205192.031598:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205192.031599:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205192.031600:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205192.031601:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205192.031602:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205192.031604:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205192.031605:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205192.031606:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205192.031607:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205192.031608:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031610:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205192.031614:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205192.031615:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205192.031618:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205192.031619:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.860190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205192.860193:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205192.860195:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205192.860198:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205192.860200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.860200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205192.860201:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.860204:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205192.860208:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205192.860209:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205192.860210:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205192.860212:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205192.860214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205192.860216:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205192.860216:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205192.860220:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205192.860221:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205192.860224:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205192.860225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205192.860228:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205192.860229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205192.860231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205192.860233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205192.860234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205192.860241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205192.860243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205192.860272:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788205192.860275:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788205192.860278:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205193.372260:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321552134, at 4321552192 00000100:00000010:3.0:1788205193.372282:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647840. 00000100:00000040:2.0:1788205193.372359:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321552134, at 4321552192 00000100:00000010:2.0:1788205193.372373:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647660. 00000100:00000001:3.0:1788205193.884195:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205193.884197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205193.884202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205193.884204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205193.884207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205193.884211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205193.884212:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205193.884215:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205193.884217:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205193.884222:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205193.884223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205193.884225:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205193.884227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205193.884229:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205193.884234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205193.884234:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205193.884238:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205193.884292:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205193.884297:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205193.884299:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205193.884302:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205193.884305:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205193.884307:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205193.884310:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205193.884312:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205193.884314:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205193.884326:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205193.884340:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205193.884345:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205194.908567:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205194.908576:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908584:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205194.908588:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205194.908591:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205194.908595:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908600:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205194.908603:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908606:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908610:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205194.908611:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205194.908615:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205194.908620:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205194.908621:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205194.908623:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205194.908625:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908627:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908635:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205194.908637:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205194.908638:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205194.908643:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205194.908646:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205194.908648:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205194.908654:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205194.908655:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205194.908661:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205194.908662:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205194.908665:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205194.908666:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205194.908668:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205194.908672:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908674:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205194.908677:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908680:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205194.908682:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205194.908728:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205194.908732:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205194.908735:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205194.908890:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205194.908894:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205194.908895:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205194.908898:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205194.908900:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205194.908903:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205194.908905:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205194.908907:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205194.908909:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.932224:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205195.932224:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205195.932232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205195.932235:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205195.932236:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205195.932239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205195.932239:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.932240:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205195.932248:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205195.932249:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205195.932251:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205195.932253:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205195.932255:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205195.932258:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.932259:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205195.932263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205195.932264:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205195.932265:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205195.932266:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205195.932270:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205195.932272:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205195.932275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205195.932278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205195.932282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205195.932284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205195.932286:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205195.932323:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205195.932327:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205195.932330:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205196.956718:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205196.956728:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205196.956731:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205196.956748:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205196.956753:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205196.956757:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.956760:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205196.956763:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.956767:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205196.957610:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205196.957615:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205196.957620:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205196.957622:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205196.957625:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205196.957627:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205196.957631:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205196.957632:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205196.957826:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205196.957831:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205196.957832:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205196.957835:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205196.957837:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205196.957839:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.957842:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205196.957844:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205196.957846:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205196.961414:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205196.961422:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205196.961428:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205197.148257:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205197.148272:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205197.148275:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205197.148279:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205197.148285:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205197.148294:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26883 00000400:00000200:3.0:1788205197.148307:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205197.148316:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205197.148320:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205197.148323:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148327:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.148331:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148334:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205197.148339:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205197.148342:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205197.148345:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205197.148350:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205197.148354:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205197.148356:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205197.148360:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148362:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.148364:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148367:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205197.148370:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205197.148383:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148386:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205197.148389:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205197.148393:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205197.148396:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148398:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205197.148401:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205197.148415:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205197.148419:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205197.148423:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205197.148425:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148429:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205197.148432:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205197.148437:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205197.148447:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412c00. 00000100:00000001:3.0:1788205197.148455:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205197.148466:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205197.148478:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205197.148484:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205197.148489:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205197.148523:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205197.148527:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205197.148531:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205197.148532:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205197.148534:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205197.148537:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205197.148540:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26883 00000400:00000200:3.0:1788205197.148544:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205197.148548:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205197.148550:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205197.148553:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148555:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.148557:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148560:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205197.148562:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205197.148563:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205197.148565:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205197.148568:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205197.148570:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205197.148572:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205197.148575:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148577:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.148579:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148580:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205197.148582:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205197.148585:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148587:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205197.148590:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205197.148597:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205197.148600:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148602:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205197.148607:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205197.148610:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148612:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205197.148616:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000020:00000040:3.0:1788205197.148618:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205197.148620:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205197.148622:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148623:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205197.148625:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205197.148626:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205197.148630:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411c00. 00000100:00000001:3.0:1788205197.148633:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205197.148639:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205197.148646:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205197.148648:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205197.148651:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205197.148659:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205197.148662:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205197.148665:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205197.148667:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205197.148668:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205197.148671:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205197.148674:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26883 00000400:00000200:3.0:1788205197.148677:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205197.148680:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205197.148683:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205197.148686:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148688:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.148689:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148691:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205197.148694:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205197.148695:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205197.148697:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205197.148699:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205197.148701:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205197.148703:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205197.148706:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148708:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.148709:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148711:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205197.148714:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205197.148716:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205197.148719:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205197.148721:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205197.148724:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205197.148726:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148727:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205197.148729:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205197.148732:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148733:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205197.148737:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205197.148739:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205197.148741:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205197.148742:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148744:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205197.148745:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205197.148747:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205197.148750:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412000. 00000100:00000001:3.0:1788205197.148754:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205197.148758:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205197.148765:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205197.148767:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205197.148771:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205197.148778:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205197.148781:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205197.148784:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788205197.148786:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205197.148790:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205197.148792:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788205197.148792:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205197.148797:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788205197.148799:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205197.148806:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26883 00000100:00000040:1.0:1788205197.148810:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205197.148813:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205197.148820:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205197.148825:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205197.148829:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1788205197.148831:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205197.148834:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148835:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.148839:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205197.148840:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788205197.148842:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:1.0:1788205197.148846:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788205197.148847:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205197.148850:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 02000000:00000001:1.0:1788205197.148852:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205197.148854:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205197.148858:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00100000:1.0:1788205197.148860:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543522944:192.168.201.118@tcp:250:kworker.0 00000020:00000001:3.0:1788205197.148861:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205197.148867:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205197.148869:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1788205197.148873:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205197.148877:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205197.148877:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.148882:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205197.148883:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148887:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:1.0:1788205197.148889:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1788205197.148892:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205197.148897:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000010:1.0:1788205197.148899:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 00000100:00000001:3.0:1788205197.148899:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.148905:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205197.148905:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205197.148910:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205197.148913:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.148918:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205197.148918:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205197.148923:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000400:00000010:1.0:1788205197.148925:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000008:00000001:3.0:1788205197.148927:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205197.148936:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205197.148943:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205197.148947:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543522944, portal 25 00000100:00000001:1.0:1788205197.148951:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205197.148953:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000010:3.0:1788205197.148956:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000100:00000040:1.0:1788205197.148960:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205197.148961:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205197.148967:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205197.148970:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.148974:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205197.148977:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788205197.148978:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205197.148982:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205197.148983:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205197.148989:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 02000000:00000010:3.0:1788205197.148991:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000200:1.0:1788205197.148993:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543522944, offset 0 00000100:00000001:3.0:1788205197.148996:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205197.149004:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205197.149005:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205197.149014:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205197.149017:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.149019:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205197.149025:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205197.149031:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205197.149039:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1788205197.149043:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.149046:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00080000:3.0:1788205197.149048:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205197.149054:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1788205197.149055:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205197.149059:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1788205197.149061:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00080000:3.0:1788205197.149063:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205197.149068:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205197.149069:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471434:471434:256:4294967295] 192.168.201.118@tcp LPNI seq info [471293:471293:8:4294967295] 00000100:00080000:3.0:1788205197.149073:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26883 00000400:00000200:1.0:1788205197.149080:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205197.149084:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205197.149121:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205197.149125:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205197.149128:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205197.149132:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.149139:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000200:1.0:1788205197.149140:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788205197.149142:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205197.149148:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205197.149151:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000800:00000200:1.0:1788205197.149153:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000020:00000001:3.0:1788205197.149154:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205197.149159:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000800:00000010:1.0:1788205197.149162:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000020:00000001:3.0:1788205197.149162:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205197.149168:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205197.149172:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.0:1788205197.149176:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205197.149177:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205197.149182:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.0:1788205197.149185:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1788205197.149186:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205197.149191:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205197.149196:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000200:1.1:1788205197.149199:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205197.149200:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205197.149205:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1788205197.149206:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 02000000:00000001:3.0:1788205197.149209:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205197.149214:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788205197.149218:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000100:00000001:3.0:1788205197.149218:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205197.149221:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000800:00000010:1.0:1788205197.149223:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000008:00000001:3.0:1788205197.149225:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.149228:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205197.149233:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000400:00000200:1.0:1788205197.149234:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1788205197.149236:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205197.149241:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205197.149245:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.149246:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788205197.149248:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205197.149251:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205197.149254:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.149255:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 02000000:00000010:3.0:1788205197.149259:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000400:00000010:1.0:1788205197.149260:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205197.149265:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.149270:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1788205197.149273:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205197.149279:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205197.149285:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205197.149290:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205197.149298:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149299:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.149307:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205197.149312:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205197.149321:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26893) 00000100:00000001:1.0:1788205197.149322:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.149326:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205197.149335:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.149339:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205197.149343:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321557254 00000100:00000010:1.0:1788205197.149349:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647960. 00000100:00000001:1.0:1788205197.149359:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205197.149366:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321557254 00000100:00000010:1.0:1788205197.149370:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16472a0. 00000100:00000001:1.0:1788205197.149384:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205197.149388:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205197.149391:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.149394:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205197.149398:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205197/real 1788205197] req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205197.149415:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205197.149417:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205197.149419:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205197.149422:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205197.149423:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205197.149426:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205197.149430:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205197.149438:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149445:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205197.149446:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.149448:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.149450:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205197.149454:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149461:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205197.149466:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149472:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205197.149475:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.149477:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.149478:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.149481:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205197.149483:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205197.149486:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205197.149490:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205197.149493:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205197.149497:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205197.149505:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543522944:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205197.149510:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205197.149513:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149520:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205197.149521:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.149524:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205197.149526:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205197.149528:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205197.149530:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523072:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205197.149533:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205197.149536:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205197.149537:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.149539:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205197.149543:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205197.149544:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.149548:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205197.149551:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205197.149554:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205197.149558:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543523072, portal 10 00000100:00000001:1.0:1788205197.149560:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205197.149561:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788205197.149568:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149575:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205197.149577:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205197.149579:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205197.149581:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543523072, offset 0 00000400:00000200:1.0:1788205197.149585:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205197.149589:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205197.149593:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205197.149596:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205197.149598:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205197.149600:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205197.149603:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205197.149605:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471435:471435:256:4294967295] 192.168.201.118@tcp LPNI seq info [471294:471294:8:4294967295] 00000400:00000200:1.0:1788205197.149610:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205197.149615:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205197.149619:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205197.149623:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205197.149626:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205197.149629:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788205197.149633:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205197.149636:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1788205197.149640:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205197.149641:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205197.149644:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205197.149648:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205197.149651:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205197.149653:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205197.149655:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205197.149659:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149667:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.149671:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149677:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.149679:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205197.149682:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205197.149686:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205197.149689:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205197.149692:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321555987 00000100:00000001:1.0:1788205197.149695:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321557254 00000100:00000001:1.0:1788205197.149697:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321555987 00000100:00000001:1.0:1788205197.149699:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321557254 00000100:00000001:1.0:1788205197.149701:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205197.149703:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.149705:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205197.149708:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205197/real 1788205197] req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205197.149716:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205197.149717:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205197.149719:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205197.149721:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205197.149723:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205197.149724:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205197.149727:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205197.149735:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149741:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205197.149742:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.149744:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.149745:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205197.149748:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149754:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205197.149758:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149764:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205197.149768:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.149769:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.149770:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.149771:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205197.149774:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205197.149776:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205197.149779:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205197.149782:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205197.149785:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205197.149792:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523072:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205197.149796:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205197.149799:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149806:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205197.149807:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.149809:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205197.149811:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205197.149813:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205197.149815:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523200:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205197.149818:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205197.149820:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205197.149821:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.149823:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205197.149827:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205197.149828:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.149831:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205197.149834:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205197.149836:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205197.149839:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543523200, portal 10 00000100:00000001:1.0:1788205197.149841:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205197.149843:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:1.0:1788205197.149848:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149854:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205197.149857:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205197.149859:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205197.149861:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543523200, offset 0 00000400:00000200:1.0:1788205197.149864:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205197.149868:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205197.149871:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205197.149874:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205197.149877:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205197.149879:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205197.149881:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205197.149884:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471436:471436:256:4294967295] 192.168.201.118@tcp LPNI seq info [471295:471295:8:4294967295] 00000400:00000200:1.0:1788205197.149888:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205197.149893:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205197.149897:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205197.149900:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205197.149904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205197.149906:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788205197.149910:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205197.149912:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1788205197.149916:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205197.149917:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205197.149920:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205197.149923:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205197.149926:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205197.149928:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205197.149930:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205197.149934:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149946:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.149950:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.149956:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.149957:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205197.149960:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205197.149963:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205197.149966:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205197.149967:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321555988 00000100:00000001:1.0:1788205197.149970:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321557254 00000100:00000001:1.0:1788205197.149972:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321555988 00000100:00000001:1.0:1788205197.149974:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321557254 00000100:00000001:1.0:1788205197.149976:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205197.149977:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.149979:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205197.149982:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205197/real 1788205197] req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205197.149989:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205197.149990:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205197.149992:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205197.149993:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205197.149995:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205197.149997:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205197.150000:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205197.150007:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.150013:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205197.150015:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.150016:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.150018:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205197.150021:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.150027:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205197.150030:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.150037:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205197.150039:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.150040:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.150041:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.150043:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205197.150045:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205197.150047:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205197.150050:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205197.150053:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205197.150056:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205197.150063:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523200:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205197.150067:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.150069:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.150072:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543522944/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.150079:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205197.150082:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205197.150085:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205197.150088:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205197.150595:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205197.150600:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205197.150603:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205197.150606:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:1.0:1788205197.150613:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205197.150615:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205197.150617:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205197.150622:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205197.150624:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.150625:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.150631:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543523072/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.150638:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205197.150639:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205197.150641:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205197.150644:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205197.150645:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205197.150646:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205197.150648:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205197.150650:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788205197.150652:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205197.150653:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205197.150655:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205197.150658:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205197.150659:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.150660:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.150664:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543523200/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.150670:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205197.150671:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205197.150673:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205197.150675:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205197.150676:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205197.150677:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205197.150679:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205197.150681:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788205197.150683:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205197.150684:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205197.150686:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205197.150689:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205197.150690:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.150692:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.150703:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205197.150705:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.150708:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205197.150710:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205197.150713:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205197.150717:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152487:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205197.152491:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.152498:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205197.152503:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205197.152506:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205197.152514:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523328:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205197.152519:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205197.152525:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205197.152528:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.152531:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205197.152542:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205197.152544:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.152552:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205197.152557:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205197.152567:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205197.152578:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543523328, portal 4 00000100:00000001:1.0:1788205197.152580:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205197.152582:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205197.152592:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152600:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205197.152602:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205197.152606:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205197.152610:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543523328, offset 0 00000400:00000200:1.0:1788205197.152617:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205197.152628:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205197.152638:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205197.152645:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205197.152649:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205197.152653:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205197.152656:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205197.152660:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471437:471437:256:4294967295] 192.168.201.118@tcp LPNI seq info [471296:471296:8:4294967295] 00000400:00000200:1.0:1788205197.152664:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205197.152675:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205197.152681:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205197.152687:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205197.152694:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205197.152698:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788205197.152705:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205197.152710:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1788205197.152715:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205197.152717:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205197.152723:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205197.152728:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205197.152734:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205197.152737:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205197.152741:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205197.152746:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152757:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.152760:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152767:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.152769:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205197.152777:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205197.152782:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205197.152785:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205197.152789:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321556000 00000100:00000001:1.0:1788205197.152793:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321557254 00000100:00000001:1.0:1788205197.152795:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321556000 00000100:00000001:1.0:1788205197.152797:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321557254 00000100:00000001:1.0:1788205197.152799:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205197.152802:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.152806:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205197.152810:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205197/real 1788205197] req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205197.152820:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205197.152821:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205197.152824:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205197.152827:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205197.152828:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205197.152830:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205197.152834:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205197.152842:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152848:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205197.152850:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.152853:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.152855:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205197.152860:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152868:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205197.152872:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152881:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205197.152884:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.152885:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.152887:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.152890:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205197.152894:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205197.152898:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205197.152903:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205197.152906:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205197.152910:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205197.152919:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523328:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205197.152925:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205197.152931:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152938:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205197.152939:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.152942:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205197.152943:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205197.152945:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205197.152948:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523456:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205197.152952:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205197.152954:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205197.152956:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205197.152958:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205197.152962:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205197.152964:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205197.152968:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205197.152971:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205197.152974:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205197.152978:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543523456, portal 4 00000100:00000001:1.0:1788205197.152980:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205197.152981:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205197.152987:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.152994:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205197.152996:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205197.152999:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205197.153001:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543523456, offset 0 00000400:00000200:1.0:1788205197.153005:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205197.153009:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205197.153013:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205197.153016:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205197.153019:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205197.153021:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205197.153023:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205197.153026:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471438:471438:256:4294967295] 192.168.201.118@tcp LPNI seq info [471297:471297:8:4294967295] 00000400:00000200:1.0:1788205197.153031:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205197.153037:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205197.153042:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205197.153045:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205197.153049:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205197.153052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1788205197.153057:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205197.153060:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1788205197.153064:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205197.153065:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205197.153068:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205197.153071:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205197.153074:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205197.153076:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205197.153078:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205197.153082:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.153108:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.153111:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205222 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.153119:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.153120:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205197.153124:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205197.153126:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205197.153130:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205197.153132:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321556024 00000100:00000001:1.0:1788205197.153134:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321557254 00000100:00000001:1.0:1788205197.153137:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321556024 00000100:00000001:1.0:1788205197.153139:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321557254 00000100:00000001:1.0:1788205197.153141:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205197.153143:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.153145:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205197.153149:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205197/real 1788205197] req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205197.153157:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205197.153158:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205197.153160:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205197.153162:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205197.153164:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205197.153166:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205197.153170:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205197.153178:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.153190:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205197.153192:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.153194:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.153195:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205197.153199:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.153205:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205197.153209:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.153216:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205197.153218:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.153220:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205197.153221:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.153223:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205197.153225:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205197.153227:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205197.153231:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205197.153233:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205197.153237:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205197.153244:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523456:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205197.153248:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.153251:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.153254:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543523328/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.153261:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205197.153264:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205197.153267:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205197.153270:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205197.153272:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205197.153274:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205197.153276:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205197.153279:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205197.153283:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205197.153285:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205197.153286:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205197.153289:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205197.153291:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.153292:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205197.153295:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543523456/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205222 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205197.153301:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205197.153302:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205197.153304:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205197.153306:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205197.153307:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205197.153308:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205197.153309:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205197.153311:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205197.153313:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205197.153314:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205197.153315:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205197.153317:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205197.153318:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.153321:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.153330:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205197.153332:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205197.153335:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205197.153337:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205197.980246:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205197.980255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205197.980263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205197.980265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205197.980268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205197.980269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205197.980273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205197.980274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205197.980357:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205197.980377:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205197.980379:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205197.980384:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205197.980388:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205197.980391:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205197.980394:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205197.980397:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205197.980400:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205197.981750:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205197.981756:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205197.981761:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205197.981775:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205197.981780:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205197.981782:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205197.981786:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205197.981789:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205197.981792:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.981794:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205197.981797:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205197.981800:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205198.492248:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321557254, at 4321557312 00000100:00000010:2.0:1788205198.492267:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647960. 00000100:00000040:3.0:1788205198.492504:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321557254, at 4321557312 00000100:00000010:3.0:1788205198.492522:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16472a0. 00000100:00000001:1.0:1788205199.004214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205199.004223:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205199.004225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205199.004230:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205199.004234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205199.004239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205199.004242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205199.004245:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205199.004249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205199.004325:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205199.004331:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205199.004338:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205199.004452:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205199.004458:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205199.004459:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205199.004463:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205199.004467:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205199.004470:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.004474:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205199.004477:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205199.004480:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205199.004517:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205199.004524:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205199.004525:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205199.004529:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205199.004531:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205199.004742:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205199.004747:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205199.004751:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205200.028218:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205200.028230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205200.028230:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205200.028230:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205200.028234:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205200.028242:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:1.0:1788205200.028243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205200.028244:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205200.028248:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:1.0:1788205200.028249:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205200.028256:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205200.028258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205200.028260:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205200.028263:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:1.0:1788205200.028266:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205200.028267:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205200.028268:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205200.028273:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205200.028274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205200.028274:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205200.028280:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205200.028283:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205200.028284:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205200.028286:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205200.028288:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205200.028296:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205200.028296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205200.028300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205200.028305:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205200.028308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205200.028315:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205200.028317:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205200.028416:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205200.028421:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205200.028425:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205201.052205:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205201.052207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205201.052212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205201.052214:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205201.052217:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205201.052218:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205201.052220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205201.052224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205201.052225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205201.052225:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205201.052229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205201.052231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205201.052231:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205201.052236:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.052240:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205201.052243:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.052245:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205201.052368:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205201.052373:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205201.052376:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205201.052394:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205201.052398:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205201.052400:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205201.052403:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205201.052406:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205201.052407:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.052410:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205201.052412:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205201.052414:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205202.076206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205202.076208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205202.076213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.076216:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205202.076218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205202.076220:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205202.076223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205202.076223:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205202.076226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.076228:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205202.076229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205202.076230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205202.076233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205202.076234:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205202.076235:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205202.076240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205202.076241:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205202.076246:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205202.076247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.076249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.076254:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205202.076262:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205202.076266:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.076268:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205202.076271:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205202.076274:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205202.076311:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205202.076315:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205202.076319:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205202.268222:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205202.268230:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205202.268232:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205202.268235:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205202.268238:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205202.268243:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26888 00000400:00000200:2.0:1788205202.268250:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205202.268256:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205202.268258:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205202.268260:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268262:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268264:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268266:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205202.268268:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205202.268269:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205202.268271:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205202.268274:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205202.268275:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205202.268277:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205202.268279:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268280:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268281:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268306:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205202.268308:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205202.268311:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268313:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205202.268315:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205202.268317:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205202.268320:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268321:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205202.268323:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205202.268330:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788205202.268333:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205202.268336:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205202.268337:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268345:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205202.268349:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205202.268356:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205202.268364:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788205202.268371:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205202.268380:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205202.268390:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205202.268394:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205202.268399:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205202.268424:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205202.268428:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205202.268431:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205202.268433:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205202.268434:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205202.268436:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205202.268454:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26888 00000400:00000200:2.0:1788205202.268458:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205202.268460:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205202.268461:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205202.268463:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268465:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268466:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268467:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205202.268469:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205202.268469:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788205202.268471:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205202.268472:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788205202.268473:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788205202.268474:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788205202.268476:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268477:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268478:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268480:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205202.268481:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788205202.268483:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268484:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205202.268485:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205202.268489:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205202.268490:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268491:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205202.268494:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205202.268496:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268497:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205202.268500:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 00000020:00000040:2.0:1788205202.268501:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205202.268503:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205202.268504:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268505:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205202.268506:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205202.268507:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205202.268509:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788205202.268512:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205202.268515:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205202.268520:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205202.268521:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205202.268523:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205202.268528:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205202.268530:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205202.268532:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788205202.268533:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205202.268534:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205202.268535:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205202.268536:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26888 00000400:00000200:2.0:1788205202.268538:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205202.268540:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205202.268541:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205202.268543:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268544:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268545:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268546:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205202.268547:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205202.268548:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788205202.268549:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205202.268551:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788205202.268552:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788205202.268553:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788205202.268555:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268556:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268557:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268558:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205202.268559:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788205202.268560:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268561:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205202.268562:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205202.268563:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205202.268564:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268565:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205202.268567:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205202.268568:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268569:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205202.268570:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788205202.268571:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205202.268572:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205202.268573:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268574:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205202.268575:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205202.268575:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205202.268577:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788205202.268579:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205202.268581:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205202.268584:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205202.268586:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205202.268588:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205202.268592:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205202.268594:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205202.268596:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205202.268596:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205202.268597:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205202.268599:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205202.268600:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26888 00000400:00000200:2.0:1788205202.268602:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205202.268603:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205202.268604:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205202.268605:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268606:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268607:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268608:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205202.268609:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205202.268610:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788205202.268611:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205202.268612:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788205202.268613:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205202.268614:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788205202.268616:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268617:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268619:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268620:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205202.268622:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788205202.268624:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268627:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205202.268629:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205202.268631:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205202.268632:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268634:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205202.268636:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788205202.268638:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268639:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205202.268643:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788205202.268645:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205202.268647:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205202.268648:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268649:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205202.268651:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205202.268653:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205202.268656:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788205202.268659:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205202.268663:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205202.268671:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205202.268672:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205202.268674:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205202.268679:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205202.268680:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205202.268682:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205202.268683:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205202.268684:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205202.268687:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205202.268688:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26888 00000400:00000200:2.0:1788205202.268690:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205202.268692:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205202.268693:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205202.268695:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268696:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268697:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268698:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205202.268700:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205202.268700:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788205202.268701:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205202.268702:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788205202.268703:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788205202.268704:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788205202.268706:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268707:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205202.268708:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268709:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205202.268711:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788205202.268712:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205202.268714:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205202.268714:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205202.268716:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205202.268717:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268718:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205202.268719:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788205202.268720:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268721:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205202.268723:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788205202.268725:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205202.268727:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205202.268727:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205202.268728:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205202.268729:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205202.268730:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205202.268732:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788205202.268747:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205202.268749:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205202.268754:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205202.268755:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205202.268757:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205202.268761:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205202.268764:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26898) 00000100:00000001:1.0:1788205202.268816:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205202.268819:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205202.268821:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205202.268825:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.268830:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205202.268832:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.268835:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205202.268837:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205202.268838:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205202.268840:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543523584:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205202.268843:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205202.268846:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205202.268848:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.268850:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205202.268854:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205202.268856:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205202.268860:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205202.268863:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205202.268869:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205202.268874:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543523584, portal 25 00000100:00000001:1.0:1788205202.268875:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205202.268876:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000100:00000040:1.0:1788205202.268879:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.268883:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205202.268885:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205202.268887:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205202.268889:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543523584, offset 0 00000400:00000200:1.0:1788205202.268893:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205202.268899:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.268903:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205202.268907:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205202.268910:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205202.268913:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205202.268914:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205202.268928:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471439:471439:256:4294967295] 192.168.201.118@tcp LPNI seq info [471298:471298:8:4294967295] 00000400:00000200:1.0:1788205202.268931:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.268942:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205202.268947:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.268950:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205202.268955:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205202.268958:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205202.268961:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205202.268964:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205202.268967:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.268968:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205202.268973:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205202.268976:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205202.268979:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205202.268981:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205202.268984:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205202.268986:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.268992:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.268994:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.268999:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.268999:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205202.269003:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205202.269005:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205202.269008:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321562373 00000100:00000010:1.0:1788205202.269011:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647300. 00000100:00000001:1.0:1788205202.269015:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205202.269019:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321562373 00000100:00000010:1.0:1788205202.269021:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16479c0. 00000100:00000001:1.0:1788205202.269023:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205202.269024:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205202.269026:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.269028:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205202.269030:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205202/real 1788205202] req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205202.269040:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205202.269042:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205202.269050:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205202.269052:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205202.269053:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205202.269054:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205202.269056:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205202.269063:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269069:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205202.269071:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269073:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.269075:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205202.269079:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269085:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205202.269112:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269120:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205202.269123:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269125:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.269126:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269129:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205202.269131:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205202.269134:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205202.269138:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205202.269140:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205202.269145:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205202.269153:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543523584:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205202.269158:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205202.269161:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269167:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205202.269168:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.269170:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205202.269172:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205202.269173:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205202.269176:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523712:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205202.269178:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205202.269180:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205202.269181:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.269182:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205202.269184:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205202.269185:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205202.269188:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205202.269189:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205202.269192:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205202.269194:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543523712, portal 10 00000100:00000001:1.0:1788205202.269195:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205202.269196:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000040:1.0:1788205202.269200:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269204:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205202.269206:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205202.269207:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205202.269208:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543523712, offset 0 00000400:00000200:1.0:1788205202.269210:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205202.269213:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.269216:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205202.269218:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205202.269219:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205202.269221:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205202.269222:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205202.269224:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471440:471440:256:4294967295] 192.168.201.118@tcp LPNI seq info [471299:471299:8:4294967295] 00000400:00000200:1.0:1788205202.269227:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.269231:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205202.269234:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.269235:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205202.269238:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205202.269240:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205202.269242:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205202.269244:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205202.269246:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.269247:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205202.269249:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205202.269251:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205202.269253:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205202.269254:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205202.269256:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205202.269260:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269266:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.269268:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269272:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.269273:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205202.269275:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205202.269278:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205202.269280:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205202.269281:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321561107 00000100:00000001:1.0:1788205202.269297:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321562373 00000100:00000001:1.0:1788205202.269299:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321561107 00000100:00000001:1.0:1788205202.269301:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321562373 00000100:00000001:1.0:1788205202.269302:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205202.269303:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.269305:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205202.269307:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205202/real 1788205202] req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205202.269312:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205202.269313:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205202.269314:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205202.269316:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205202.269317:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205202.269318:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205202.269320:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205202.269325:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269328:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205202.269329:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269329:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.269330:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205202.269332:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269335:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205202.269338:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269343:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205202.269346:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269348:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.269349:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269351:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205202.269353:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205202.269356:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205202.269359:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205202.269361:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205202.269365:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205202.269373:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523712:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205202.269376:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205202.269379:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269385:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205202.269386:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.269388:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205202.269389:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205202.269391:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205202.269394:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523840:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205202.269396:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205202.269398:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205202.269399:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.269401:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205202.269403:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205202.269405:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205202.269408:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205202.269410:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205202.269413:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205202.269415:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543523840, portal 10 00000100:00000001:1.0:1788205202.269417:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205202.269418:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 00000100:00000040:1.0:1788205202.269423:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269429:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205202.269431:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205202.269433:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205202.269435:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543523840, offset 0 00000400:00000200:1.0:1788205202.269438:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205202.269442:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.269445:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205202.269447:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205202.269449:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205202.269451:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205202.269453:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205202.269456:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471441:471441:256:4294967295] 192.168.201.118@tcp LPNI seq info [471300:471300:8:4294967295] 00000400:00000200:1.0:1788205202.269459:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.269464:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205202.269468:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.269472:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205202.269475:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205202.269477:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205202.269481:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205202.269483:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205202.269486:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.269487:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205202.269490:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205202.269493:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205202.269495:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205202.269497:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205202.269499:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205202.269502:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269512:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.269515:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269520:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.269521:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205202.269523:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205202.269526:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205202.269528:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205202.269530:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321561108 00000100:00000001:1.0:1788205202.269532:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321562373 00000100:00000001:1.0:1788205202.269534:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321561108 00000100:00000001:1.0:1788205202.269536:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321562373 00000100:00000001:1.0:1788205202.269538:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205202.269540:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.269541:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205202.269545:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205202/real 1788205202] req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205202.269551:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205202.269553:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205202.269554:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205202.269555:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205202.269557:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205202.269558:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205202.269561:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205202.269568:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269573:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205202.269575:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269576:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.269577:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205202.269580:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269586:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205202.269589:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269594:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205202.269596:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269598:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.269599:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269601:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205202.269602:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205202.269605:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205202.269608:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205202.269610:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205202.269613:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205202.269620:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523840:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205202.269623:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205202.269626:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269632:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205202.269633:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.269635:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205202.269636:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205202.269638:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205202.269640:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523968:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205202.269643:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205202.269645:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205202.269646:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.269647:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205202.269652:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205202.269654:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205202.269657:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205202.269659:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205202.269661:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205202.269664:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543523968, portal 4 00000100:00000001:1.0:1788205202.269665:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205202.269666:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331919360 : -109424377632256 : ffff9c7aa6d76a00) 00000100:00000040:1.0:1788205202.269671:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269676:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205202.269678:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205202.269680:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205202.269682:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543523968, offset 0 00000400:00000200:1.0:1788205202.269685:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205202.269688:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.269690:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205202.269693:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205202.269695:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205202.269697:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205202.269699:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205202.269702:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471442:471442:256:4294967295] 192.168.201.118@tcp LPNI seq info [471301:471301:8:4294967295] 00000400:00000200:1.0:1788205202.269705:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.269710:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205202.269713:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.269716:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205202.269719:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205202.269722:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205202.269725:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205202.269727:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205202.269730:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.269732:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205202.269734:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205202.269737:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205202.269739:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205202.269741:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205202.269743:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205202.269746:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269752:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.269754:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269760:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.269762:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205202.269764:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205202.269767:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205202.269769:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205202.269771:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321561117 00000100:00000001:1.0:1788205202.269773:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321562373 00000100:00000001:1.0:1788205202.269775:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321561117 00000100:00000001:1.0:1788205202.269777:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321562373 00000100:00000001:1.0:1788205202.269779:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205202.269780:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.269782:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205202.269785:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205202/real 1788205202] req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205202.269791:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205202.269793:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205202.269794:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205202.269796:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205202.269797:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205202.269799:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205202.269802:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205202.269810:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269815:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205202.269816:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269817:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.269819:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205202.269822:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269828:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205202.269831:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269836:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205202.269838:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269840:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.269841:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.269842:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205202.269844:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205202.269846:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205202.269849:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205202.269850:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205202.269854:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205202.269865:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543523968:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205202.269868:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205202.269871:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269876:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205202.269877:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.269879:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205202.269880:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205202.269882:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205202.269884:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524096:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205202.269887:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205202.269888:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205202.269890:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205202.269891:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205202.269894:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205202.269895:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205202.269898:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205202.269900:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205202.269903:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205202.269905:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543524096, portal 4 00000100:00000001:1.0:1788205202.269907:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205202.269908:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331914880 : -109424377636736 : ffff9c7aa6d75880) 00000100:00000040:1.0:1788205202.269912:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269917:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205202.269919:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205202.269921:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205202.269923:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543524096, offset 0 00000400:00000200:1.0:1788205202.269926:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205202.269930:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.269932:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205202.269935:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205202.269937:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205202.269939:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205202.269941:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205202.269943:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471443:471443:256:4294967295] 192.168.201.118@tcp LPNI seq info [471302:471302:8:4294967295] 00000400:00000200:1.0:1788205202.269947:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205202.269952:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205202.269956:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.269958:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205202.269961:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205202.269963:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1788205202.269967:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205202.269969:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1788205202.269972:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205202.269974:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205202.269976:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205202.269979:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205202.269981:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205202.269983:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205202.269985:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205202.269987:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.269994:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.269996:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205227 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270002:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.270003:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205202.270005:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205202.270008:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205202.270011:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205202.270013:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321561140 00000100:00000001:1.0:1788205202.270015:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321562373 00000100:00000001:1.0:1788205202.270017:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321561140 00000100:00000001:1.0:1788205202.270019:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321562373 00000100:00000001:1.0:1788205202.270021:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205202.270022:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.270024:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205202.270026:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205202/real 1788205202] req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205202.270033:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205202.270034:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205202.270036:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205202.270038:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205202.270039:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205202.270041:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205202.270043:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205202.270050:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270056:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205202.270057:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270058:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.270060:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205202.270063:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270069:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205202.270072:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270078:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205202.270080:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270082:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205202.270083:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270085:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205202.270086:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205202.270088:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205202.270107:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205202.270109:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205202.270113:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205202.270120:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524096:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205202.270123:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270125:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.270128:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543523584/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270133:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205202.270136:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205202.270138:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205202.270141:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205202.270142:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205202.270144:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205202.270146:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205202.270148:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:1.0:1788205202.270152:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205202.270153:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205202.270155:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:1.0:1788205202.270158:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205202.270159:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270160:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.270163:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543523712/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270174:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205202.270175:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205202.270176:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205202.270178:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205202.270179:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205202.270180:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205202.270182:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205202.270183:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:1.0:1788205202.270185:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205202.270186:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205202.270187:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:1.0:1788205202.270188:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205202.270189:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270191:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.270194:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543523840/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270199:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205202.270200:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205202.270201:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205202.270203:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205202.270204:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205202.270205:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205202.270206:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205202.270208:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:1.0:1788205202.270210:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205202.270211:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205202.270212:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:1.0:1788205202.270214:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205202.270215:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270216:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.270218:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76a00 x1875055543523968/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270225:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205202.270226:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205202.270228:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205202.270230:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205202.270231:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205202.270232:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205202.270234:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205202.270235:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:1.0:1788205202.270237:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205202.270238:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205202.270239:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76a00. 00000100:00000001:1.0:1788205202.270241:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205202.270243:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270244:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205202.270248:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75880 x1875055543524096/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205227 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205202.270254:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205202.270255:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205202.270256:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205202.270258:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205202.270259:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205202.270260:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205202.270261:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205202.270262:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d800. 02000000:00000001:1.0:1788205202.270264:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205202.270265:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205202.270265:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75880. 00000100:00000001:1.0:1788205202.270267:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205202.270268:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270269:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205202.270277:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205202.270278:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205202.270281:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205202.270282:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205203.100188:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205203.100188:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205203.100192:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205203.100195:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205203.100196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205203.100197:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205203.100199:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205203.100200:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205203.100206:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205203.100206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205203.100210:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205203.100212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205203.100212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205203.100215:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205203.100216:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205203.100217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205203.100221:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205203.100223:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205203.100224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205203.100225:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205203.100227:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205203.100228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205203.100230:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205203.100234:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205203.100235:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205203.100237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205203.100254:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205203.100258:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205203.100261:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205203.612534:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321562373, at 4321562432 00000100:00000010:3.0:1788205203.612553:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. 00000100:00000040:2.0:1788205203.613357:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321562373, at 4321562433 00000100:00000010:2.0:1788205203.613374:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00000100:00000001:2.0:1788205204.124272:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205204.124272:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205204.124282:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205204.124284:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205204.124290:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205204.124294:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205204.124295:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205204.124299:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205204.124301:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205204.124303:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205204.124306:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205204.124308:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205204.124311:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205204.124312:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205204.124316:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205204.124318:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205204.124326:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205204.124327:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205204.125542:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205204.125545:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205204.125549:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205204.125550:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205204.125552:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205204.125552:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205204.125555:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205204.125556:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205204.125623:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205204.125651:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205204.125654:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205205.148253:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205205.148260:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205205.148262:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205205.148267:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205205.148270:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205205.148274:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.148277:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205205.148279:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205205.148282:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148405:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205205.148408:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148411:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205205.148414:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205205.148416:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205205.148418:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148420:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205205.148422:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148423:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148424:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148426:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205205.148427:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205205.148428:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205205.148429:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148430:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205205.148430:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148431:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148432:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148434:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205205.148435:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205205.148436:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205205.148437:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148437:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205205.148438:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148439:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205205.148440:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205205.148520:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205205.148523:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205205.148526:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205205.149212:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205205.149216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205205.149217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205205.149220:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205205.149222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205205.149224:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205205.149226:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205205.149228:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205205.149230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205206.172250:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205206.172250:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205206.172264:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205206.172264:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205206.172273:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205206.172274:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205206.172288:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205206.172289:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205206.172300:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205206.172300:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205206.172311:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.172311:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205206.172327:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205206.172327:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205206.172338:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.172338:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205206.172349:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205206.172350:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205206.173400:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205206.173406:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205206.173412:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205206.173413:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205206.173417:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205206.173418:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205206.173421:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205206.173422:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205206.173496:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205206.173501:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205206.173504:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205207.196243:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205207.196243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205207.196249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205207.196255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.196255:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205207.196255:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205207.196260:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205207.196260:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205207.196264:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205207.196267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205207.196268:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205207.196268:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205207.196270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205207.196273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.196277:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205207.196278:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205207.196278:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205207.196281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205207.196284:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205207.196285:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205207.196292:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205207.196292:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205207.196297:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205207.196297:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205207.196303:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205207.196304:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205207.197537:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205207.197544:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205207.197550:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205207.389308:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205207.389315:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205207.389317:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205207.389320:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205207.389323:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205207.389329:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26893 00000400:00000200:2.0:1788205207.389337:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205207.389354:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205207.389357:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205207.389360:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389362:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.389364:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389365:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205207.389368:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205207.389370:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205207.389372:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205207.389374:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205207.389376:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205207.389377:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205207.389379:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389380:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.389381:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389383:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205207.389384:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205207.389386:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389388:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205207.389389:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205207.389392:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205207.389394:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.389395:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205207.389402:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205207.389410:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77480. 00000020:00000040:2.0:1788205207.389412:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205207.389415:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205207.389416:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.389419:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205207.389420:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205207.389429:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205207.389437:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d800. 00000100:00000001:2.0:1788205207.389443:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205207.389450:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205207.389457:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205207.389460:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205207.389463:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dbe9e2c0] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205207.389485:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205207.389487:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205207.389489:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205207.389490:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205207.389491:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205207.389493:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205207.389494:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26893 00000400:00000200:2.0:1788205207.389497:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205207.389498:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205207.389500:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205207.389502:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389503:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.389505:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389506:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205207.389508:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205207.389508:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788205207.389509:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205207.389511:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788205207.389512:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788205207.389513:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788205207.389517:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389519:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.389522:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389524:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205207.389526:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788205207.389528:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389529:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205207.389530:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205207.389537:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788205207.389537:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205207.389544:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.389547:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205207.389548:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205207.389556:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1788205207.389556:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205207.389562:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.389564:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205207.389573:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75c00. 00000100:00000040:1.0:1788205207.389574:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205207.389575:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205207.389580:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205207.389582:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.389587:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205207.389589:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205207.389591:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205207.389598:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f000. 00000100:00000001:2.0:1788205207.389603:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.389605:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1788205207.389611:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205207.389612:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.389625:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205207.389627:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788205207.389629:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205207.389630:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.389643:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205207.389647:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000623ec114] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205207.389651:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543524224:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205207.389671:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788205207.389676:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.389678:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1788205207.389680:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788205207.389682:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205207.389684:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1788205207.389688:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788205207.389688:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205207.389698:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205207.389702:0:409030:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:1.0:1788205207.389704:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 00000100:00080000:2.0:1788205207.389706:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26893 02000000:00000001:1.0:1788205207.389713:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1788205207.389720:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205207.389724:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205207.389726:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1788205207.389729:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205207.389741:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00000040:2.0:1788205207.389741:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389754:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.389757:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389759:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000010:1.0:1788205207.389760:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000020:00000001:2.0:1788205207.389764:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205207.389767:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788205207.389769:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000200:1.0:1788205207.389771:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543524224, portal 25 00000020:00000040:2.0:1788205207.389772:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000001:1.0:1788205207.389777:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205207.389781:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922048 : -109424377629568 : ffff9c7aa6d77480) 00000020:00000001:2.0:1788205207.389782:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205207.389791:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788205207.389792:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788205207.389808:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1788205207.389810:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205207.389817:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389822:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1788205207.389823:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205207.389833:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543524224, offset 0 00000400:00000200:1.0:1788205207.389843:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000040:2.0:1788205207.389844:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389851:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205207.389854:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:2.0:1788205207.389857:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389860:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205207.389863:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788205207.389863:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205207.389870:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205207.389873:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.389875:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205207.389878:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1788205207.389881:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000008:00000001:2.0:1788205207.389882:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.389888:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205207.389894:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76680. 00000400:00000200:1.0:1788205207.389897:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1788205207.389898:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205207.389907:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205207.389907:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.389910:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205207.389911:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205207.389914:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205207.389916:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000010:2.0:1788205207.389918:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067f400. 00000100:00000001:2.0:1788205207.389924:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205207.389927:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00100000:2.0:1788205207.389934:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205207.389938:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.389949:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205207.389954:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471444:471444:256:4294967295] 192.168.201.118@tcp LPNI seq info [471303:471303:8:4294967295] 00000100:00000001:2.0:1788205207.389954:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205207.389964:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000752d1d3c] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205207.389982:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788205207.389984:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205207.389992:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205207.389994:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205207.389995:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205207.389996:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205207.389999:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205207.390007:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00080000:2.0:1788205207.390017:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26893 00000800:00000200:1.0:1788205207.390018:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.390027:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000400:00000200:2.0:1788205207.390027:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205207.390035:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205207.390039:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1788205207.390040:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000040:2.0:1788205207.390045:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.0:1788205207.390046:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000100:00000001:2.0:1788205207.390051:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.390054:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.1:1788205207.390061:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788205207.390062:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:1.1:1788205207.390071:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000020:00000001:2.0:1788205207.390072:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205207.390080:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000800:00000100:1.0:1788205207.390087:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000001:2.0:1788205207.390088:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205207.390132:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788205207.390134:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205207.390136:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788205207.390138:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.390139:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.390140:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.390141:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205207.390143:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788205207.390144:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205207.390146:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205207.390147:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205207.390150:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205207.390151:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.390152:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205207.390154:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788205207.390155:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.390156:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205207.390159:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75880. 00000020:00000040:2.0:1788205207.390160:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205207.390162:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205207.390163:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.390164:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205207.390165:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205207.390166:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205207.390169:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c000. 00000100:00000001:2.0:1788205207.390171:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205207.390173:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205207.390178:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205207.390179:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205207.390182:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009176d618] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205207.390190:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205207.390193:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205207.390207:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205207.390208:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205207.390210:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205207.390213:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205207.390215:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26893 00000400:00000200:2.0:1788205207.390219:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205207.390222:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205207.390225:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205207.390227:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.390229:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.390231:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.390233:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205207.390235:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205207.390236:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788205207.390238:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205207.390240:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:2.0:1788205207.390241:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788205207.390244:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:2.0:1788205207.390246:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.390247:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205207.390248:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205207.390250:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205207.390251:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:2.0:1788205207.390252:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205207.390253:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205207.390255:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205207.390257:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205207.390259:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.390260:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205207.390261:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788205207.390263:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.390264:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205207.390266:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d76a00. 00000020:00000040:2.0:1788205207.390267:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205207.390269:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205207.390270:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205207.390271:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205207.390272:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205207.390273:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205207.390275:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d400. 00000100:00000001:2.0:1788205207.390277:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205207.390280:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205207.390285:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205207.390286:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205207.390288:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020b66f80] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205207.390292:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205207.390295:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26903) 00000800:00000010:1.0:1788205207.391292:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205207.391300:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205207.391304:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205207.391318:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205207.391320:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205207.391325:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205207.391329:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391336:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.391338:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391344:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.391345:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205207.391352:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205207.391357:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205207.391360:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321567496 00000100:00000010:1.0:1788205207.391376:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16471e0. 00000100:00000001:1.0:1788205207.391383:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205207.391396:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321567496 00000100:00000010:1.0:1788205207.391400:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647540. 00000100:00000001:1.0:1788205207.391403:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205207.391406:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205207.391409:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.391411:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205207.391414:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205207/real 1788205207] req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205207.391422:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205207.391424:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205207.391426:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205207.391428:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205207.391429:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205207.391431:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205207.391433:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205207.391438:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391442:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205207.391443:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391445:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.391446:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205207.391448:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391452:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205207.391455:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391459:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205207.391461:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391463:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.391463:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391465:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205207.391466:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205207.391469:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205207.391471:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205207.391472:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205207.391475:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205207.391480:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543524224:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205207.391484:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391486:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.391487:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77480 x1875055543524224/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391491:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205207.391492:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205207.391494:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205207.391496:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205207.391497:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205207.391498:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205207.391500:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205207.391501:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d800. 02000000:00000001:1.0:1788205207.391504:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205207.391506:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205207.391507:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77480. 00000100:00000001:1.0:1788205207.391509:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205207.391510:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391511:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391516:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205207.391518:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.391520:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205207.391522:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205207.391524:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205207.391527:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391535:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205207.391536:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.391540:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205207.391542:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205207.391544:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205207.391548:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524352:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205207.391551:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205207.391555:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205207.391557:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.391559:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205207.391563:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205207.391565:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205207.391569:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205207.391571:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205207.391574:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205207.391578:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543524352, portal 10 00000100:00000001:1.0:1788205207.391580:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205207.391581:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331915776 : -109424377635840 : ffff9c7aa6d75c00) 00000100:00000040:1.0:1788205207.391586:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391590:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205207.391591:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205207.391593:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205207.391595:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543524352, offset 0 00000400:00000200:1.0:1788205207.391598:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205207.391602:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205207.391606:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205207.391610:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205207.391613:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205207.391615:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205207.391617:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205207.391620:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471445:471445:256:4294967295] 192.168.201.118@tcp LPNI seq info [471304:471304:8:4294967295] 00000400:00000200:1.0:1788205207.391623:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205207.391629:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205207.391633:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.391635:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205207.391640:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205207.391642:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788205207.391646:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205207.391649:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1788205207.391652:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.391653:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205207.391655:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205207.391657:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205207.391659:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205207.391660:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205207.391661:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205207.391663:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391668:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.391670:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391676:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.391678:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205207.391681:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205207.391686:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205207.391690:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205207.391693:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321566229 00000100:00000001:1.0:1788205207.391696:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321567496 00000100:00000001:1.0:1788205207.391698:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321566229 00000100:00000001:1.0:1788205207.391700:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321567496 00000100:00000001:1.0:1788205207.391705:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205207.391708:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.391709:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205207.391713:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205207/real 1788205207] req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205207.391720:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205207.391722:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205207.391723:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205207.391726:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205207.391728:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205207.391729:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205207.391732:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205207.391740:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391745:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205207.391746:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391748:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.391749:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205207.391752:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391758:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205207.391761:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391766:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205207.391768:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391769:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.391770:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391771:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205207.391773:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205207.391777:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205207.391780:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205207.391781:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205207.391785:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205207.391793:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524352:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205207.391797:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205207.391799:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391805:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205207.391807:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.391809:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205207.391812:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205207.391814:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205207.391825:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524480:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205207.391829:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205207.391831:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205207.391832:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.391834:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205207.391837:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205207.391839:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205207.391842:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205207.391845:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205207.391848:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205207.391851:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543524480, portal 10 00000100:00000001:1.0:1788205207.391853:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205207.391854:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331918464 : -109424377633152 : ffff9c7aa6d76680) 00000100:00000040:1.0:1788205207.391858:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391867:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205207.391868:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205207.391869:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205207.391871:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543524480, offset 0 00000400:00000200:1.0:1788205207.391874:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205207.391876:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205207.391879:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205207.391881:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205207.391882:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205207.391884:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205207.391885:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205207.391888:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471446:471446:256:4294967295] 192.168.201.118@tcp LPNI seq info [471305:471305:8:4294967295] 00000400:00000200:1.0:1788205207.391890:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205207.391894:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205207.391897:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.391898:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205207.391901:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205207.391903:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788205207.391906:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205207.391907:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1788205207.391909:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.391911:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205207.391913:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205207.391915:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205207.391917:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205207.391918:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205207.391919:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205207.391921:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391926:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.391928:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391935:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.391936:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205207.391938:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205207.391940:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205207.391942:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205207.391943:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321566230 00000100:00000001:1.0:1788205207.391945:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321567496 00000100:00000001:1.0:1788205207.391947:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321566230 00000100:00000001:1.0:1788205207.391948:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321567496 00000100:00000001:1.0:1788205207.391949:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205207.391951:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.391952:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205207.391954:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205207/real 1788205207] req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205207.391958:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205207.391959:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205207.391960:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205207.391961:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205207.391962:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205207.391963:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205207.391965:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205207.391970:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391973:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205207.391974:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391975:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.391976:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205207.391978:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391982:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205207.391984:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.391988:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205207.391989:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391990:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.391991:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.391992:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205207.391993:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205207.391994:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205207.391996:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205207.391997:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205207.391999:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205207.392003:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524480:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205207.392005:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205207.392007:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392012:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205207.392012:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.392013:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205207.392014:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205207.392015:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205207.392017:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524608:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205207.392019:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205207.392020:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205207.392021:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.392022:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205207.392024:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205207.392026:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205207.392027:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205207.392029:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205207.392030:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205207.392032:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543524608, portal 4 00000100:00000001:1.0:1788205207.392033:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205207.392034:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331914880 : -109424377636736 : ffff9c7aa6d75880) 00000100:00000040:1.0:1788205207.392037:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392041:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205207.392042:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205207.392043:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205207.392044:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543524608, offset 0 00000400:00000200:1.0:1788205207.392046:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205207.392048:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205207.392050:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205207.392051:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205207.392053:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205207.392054:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205207.392055:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205207.392057:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471447:471447:256:4294967295] 192.168.201.118@tcp LPNI seq info [471306:471306:8:4294967295] 00000400:00000200:1.0:1788205207.392059:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205207.392063:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205207.392065:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.392067:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205207.392069:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205207.392071:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788205207.392073:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205207.392075:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1788205207.392077:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.392079:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205207.392081:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205207.392082:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205207.392084:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205207.392085:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205207.392086:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205207.392088:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392108:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.392110:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392113:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.392114:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205207.392116:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205207.392118:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205207.392120:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205207.392121:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321566240 00000100:00000001:1.0:1788205207.392122:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321567496 00000100:00000001:1.0:1788205207.392124:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321566240 00000100:00000001:1.0:1788205207.392125:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321567496 00000100:00000001:1.0:1788205207.392126:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205207.392127:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.392128:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205207.392130:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205207/real 1788205207] req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205207.392134:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205207.392135:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205207.392136:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205207.392138:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205207.392139:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205207.392140:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205207.392142:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205207.392147:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392150:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205207.392151:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392152:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.392153:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205207.392154:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392158:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205207.392160:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392169:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205207.392171:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392173:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.392174:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392175:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205207.392177:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205207.392179:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205207.392182:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205207.392184:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205207.392187:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205207.392206:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524608:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205207.392211:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205207.392214:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392221:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205207.392222:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.392224:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205207.392226:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205207.392228:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205207.392230:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524736:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205207.392232:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205207.392234:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205207.392235:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205207.392236:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205207.392238:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205207.392239:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205207.392241:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205207.392243:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205207.392245:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205207.392247:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543524736, portal 4 00000100:00000001:1.0:1788205207.392248:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205207.392249:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331919360 : -109424377632256 : ffff9c7aa6d76a00) 00000100:00000040:1.0:1788205207.392252:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392256:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205207.392258:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205207.392259:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205207.392260:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543524736, offset 0 00000400:00000200:1.0:1788205207.392262:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205207.392265:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205207.392267:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205207.392268:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205207.392270:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205207.392271:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205207.392273:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205207.392275:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471448:471448:256:4294967295] 192.168.201.118@tcp LPNI seq info [471307:471307:8:4294967295] 00000400:00000200:1.0:1788205207.392277:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205207.392282:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205207.392284:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.392286:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205207.392288:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205207.392290:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000200:1.1:1788205207.392292:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205207.392294:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 28 secs later) 00000800:00000100:1.0:1788205207.392296:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205207.392297:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205207.392299:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205207.392301:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205207.392303:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205207.392304:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205207.392305:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205207.392307:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392312:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.392313:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205232 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392317:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.392318:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205207.392319:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205207.392321:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205207.392322:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205207.392323:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321566263 00000100:00000001:1.0:1788205207.392325:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321567496 00000100:00000001:1.0:1788205207.392326:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321566263 00000100:00000001:1.0:1788205207.392327:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321567496 00000100:00000001:1.0:1788205207.392329:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205207.392330:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.392331:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205207.392333:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205207/real 1788205207] req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205207.392338:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205207.392339:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205207.392339:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205207.392340:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205207.392341:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205207.392342:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205207.392344:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205207.392349:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392353:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205207.392353:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392354:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.392355:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205207.392357:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392363:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205207.392366:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392371:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205207.392374:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392375:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205207.392376:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392378:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205207.392379:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205207.392382:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205207.392385:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205207.392387:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205207.392391:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205207.392399:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524736:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205207.392403:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392404:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.392406:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75c00 x1875055543524352/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392410:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205207.392411:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205207.392413:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205207.392414:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205207.392415:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205207.392416:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205207.392417:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205207.392418:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f000. 02000000:00000001:1.0:1788205207.392420:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205207.392421:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205207.392422:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75c00. 00000100:00000001:1.0:1788205207.392423:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205207.392424:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392425:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.392427:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76680 x1875055543524480/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392431:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205207.392431:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205207.392433:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205207.392434:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205207.392435:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205207.392436:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205207.392438:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205207.392439:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067f400. 02000000:00000001:1.0:1788205207.392441:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205207.392442:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205207.392443:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76680. 00000100:00000001:1.0:1788205207.392446:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205207.392447:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392448:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.392451:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75880 x1875055543524608/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392457:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205207.392458:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205207.392459:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205207.392461:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205207.392463:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205207.392464:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205207.392466:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205207.392467:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c000. 02000000:00000001:1.0:1788205207.392469:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205207.392470:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205207.392472:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75880. 00000100:00000001:1.0:1788205207.392474:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205207.392475:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392476:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205207.392479:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d76a00 x1875055543524736/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205232 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205207.392483:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205207.392484:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205207.392485:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205207.392486:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205207.392487:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205207.392487:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205207.392488:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205207.392489:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d400. 02000000:00000001:1.0:1788205207.392490:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205207.392491:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205207.392492:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d76a00. 00000100:00000001:1.0:1788205207.392493:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205207.392494:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392496:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205207.392498:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205207.392499:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205207.392500:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205207.392501:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205208.220235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205208.220236:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205208.220240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205208.220247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205208.220247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205208.220248:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205208.220250:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205208.220252:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205208.220256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205208.220257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205208.220259:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205208.220260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205208.220261:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205208.220266:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205208.220268:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205208.220269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205208.220270:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205208.220274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205208.220276:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205208.220277:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205208.220279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205208.220279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205208.220282:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205208.220283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205208.220296:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205208.220300:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205208.220354:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205208.220358:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205208.220361:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205208.732214:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321567496, at 4321567552 00000100:00000010:3.0:1788205208.732231:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647540. 00000100:00000040:2.0:1788205208.732242:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321567496, at 4321567552 00000100:00000010:2.0:1788205208.732258:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00000100:00000001:3.0:1788205209.244191:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205209.244199:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205209.244201:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205209.244202:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205209.244204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205209.244207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205209.244211:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205209.244212:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205209.244213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205209.244217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205209.244219:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205209.244222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205209.244222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205209.244225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205209.244229:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205209.244230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205209.244230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205209.244233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205209.244242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205209.244243:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205209.244247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205209.244249:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205209.244254:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205209.244255:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205209.244261:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205209.244266:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205209.244326:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205209.244330:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205209.244333:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205210.268219:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205210.268227:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205210.268231:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205210.268231:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205210.268243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205210.268243:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205210.268244:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205210.268253:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205210.268255:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205210.268256:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788205210.268261:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205210.268262:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205210.268265:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205210.268268:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205210.268269:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205210.268271:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205210.268274:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205210.268277:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205210.268278:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205210.268280:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205210.268284:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205210.268286:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205210.268289:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205210.268290:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205210.268292:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205210.268301:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205210.268312:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205210.268316:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205210.268321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205210.268322:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205210.268327:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205210.268328:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205210.268458:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205210.268463:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205210.268466:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205211.292392:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205211.292400:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205211.292402:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205211.292407:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205211.292411:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205211.292415:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205211.292418:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205211.292420:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205211.292423:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205211.293453:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205211.293457:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205211.293458:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205211.293461:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205211.293463:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205211.293465:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205211.293468:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205211.293470:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205211.293472:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205211.294324:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205211.294332:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205211.294338:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205211.294340:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205211.294344:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205211.294345:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205211.294349:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205211.294350:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205211.294797:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205211.294801:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205211.294805:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205212.316238:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205212.316245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205212.316247:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205212.316250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205212.316251:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205212.316259:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205212.316260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205212.316264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205212.316266:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205212.316271:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205212.316271:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205212.316274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205212.316276:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205212.316279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205212.316279:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205212.316281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205212.316284:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.316287:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205212.316292:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205212.316293:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205212.316296:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205212.316299:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205212.316301:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205212.316304:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205212.316307:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205212.316309:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205212.318188:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205212.318193:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205212.318196:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788205212.508157:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205212.508165:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205212.508168:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205212.508171:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205212.508174:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205212.508180:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26898 00000400:00000200:2.0:1788205212.508189:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205212.508217:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205212.508221:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205212.508224:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508227:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205212.508230:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508232:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205212.508237:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205212.508238:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205212.508240:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205212.508243:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205212.508244:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205212.508246:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205212.508248:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508249:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205212.508250:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508251:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205212.508253:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205212.508255:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508256:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205212.508258:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205212.508261:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205212.508263:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508265:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205212.508266:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205212.508276:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74700. 00000020:00000040:2.0:1788205212.508278:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205212.508281:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205212.508282:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508285:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205212.508287:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205212.508290:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205212.508296:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067ec00. 00000100:00000001:2.0:1788205212.508301:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205212.508308:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205212.508314:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205212.508317:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205212.508320:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e37a50d3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205212.508338:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205212.508342:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205212.508346:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205212.508347:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205212.508348:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205212.508351:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205212.508359:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26898 00000400:00000200:2.0:1788205212.508363:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205212.508366:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205212.508367:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205212.508370:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508372:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205212.508373:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508375:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205212.508377:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205212.508379:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788205212.508381:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205212.508383:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:1.0:1788205212.508385:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788205212.508385:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788205212.508387:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205212.508390:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1788205212.508390:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205212.508392:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205212.508392:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205212.508393:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508395:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205212.508397:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000040:1.0:1788205212.508399:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205212.508399:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508402:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205212.508404:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205212.508406:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205212.508408:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205212.508409:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205212.508411:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788205212.508411:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508412:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:1.0:1788205212.508413:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205212.508415:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788205212.508416:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788205212.508418:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543524864:192.168.201.118@tcp:250:kworker.0 00000008:00000001:2.0:1788205212.508419:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508421:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.508422:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1788205212.508424:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75180. 02000000:00000001:1.0:1788205212.508426:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1788205212.508426:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205212.508428:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205212.508429:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205212.508429:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508430:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205212.508432:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788205212.508432:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205212.508433:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205212.508437:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000010:2.0:1788205212.508437:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788205212.508439:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508440:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.508444:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:2.0:1788205212.508445:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205212.508451:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205212.508453:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205212.508457:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000077ed6bac] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205212.508465:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205212.508469:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205212.508472:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1788205212.508473:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205212.508475:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205212.508478:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205212.508480:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26898 00000400:00000010:1.0:1788205212.508482:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000200:2.0:1788205212.508485:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205212.508487:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000400:00000010:1.0:1788205212.508490:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00080000:2.0:1788205212.508490:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205212.508492:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508494:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1788205212.508496:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543524864, portal 25 00000100:00000040:2.0:1788205212.508496:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205212.508498:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788205212.508498:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205212.508500:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331910400 : -109424377641216 : ffff9c7aa6d74700) 00000020:00000001:2.0:1788205212.508500:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205212.508501:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788205212.508503:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1788205212.508505:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205212.508505:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788205212.508506:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:2.0:1788205212.508509:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788205212.508512:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205212.508513:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788205212.508514:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205212.508515:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205212.508516:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205212.508518:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000001:2.0:1788205212.508518:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205212.508520:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000200:1.0:1788205212.508521:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543524864, offset 0 00000100:00080000:2.0:1788205212.508522:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508524:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205212.508526:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205212.508527:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 02000000:00000001:2.0:1788205212.508529:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205212.508531:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508533:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205212.508535:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205212.508537:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508539:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.508541:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:2.0:1788205212.508542:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77800. 00000020:00000040:2.0:1788205212.508544:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205212.508546:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1788205212.508547:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 02000000:00000001:2.0:1788205212.508548:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508549:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1788205212.508551:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:2.0:1788205212.508551:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205212.508552:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.508555:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 02000000:00000010:2.0:1788205212.508555:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:2.0:1788205212.508558:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.508559:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205212.508562:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00100000:2.0:1788205212.508562:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205212.508566:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471449:471449:256:4294967295] 192.168.201.118@tcp LPNI seq info [471308:471308:8:4294967295] 00000100:00000001:2.0:1788205212.508569:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205212.508571:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:2.0:1788205212.508571:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205212.508574:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bbe5b65e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205212.508581:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:2.0:1788205212.508582:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205212.508585:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1788205212.508586:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:2.0:1788205212.508588:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205212.508590:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:1.0:1788205212.508591:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00080000:2.0:1788205212.508591:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205212.508594:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205212.508596:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26898 00000400:00000200:2.0:1788205212.508600:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205212.508603:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205212.508605:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205212.508608:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508610:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205212.508612:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508613:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205212.508615:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205212.508617:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788205212.508618:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205212.508620:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788205212.508622:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205212.508624:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:2.0:1788205212.508627:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:1.0:1788205212.508629:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788205212.508629:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205212.508631:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.0:1788205212.508633:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000001:2.0:1788205212.508633:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205212.508635:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788205212.508637:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000800:00000200:1.1:1788205212.508638:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:2.0:1788205212.508639:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205212.508641:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1788205212.508641:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 02000000:00000001:2.0:1788205212.508644:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:1.0:1788205212.508646:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000001:2.0:1788205212.508646:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508648:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000800:00000010:1.0:1788205212.508649:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000008:00100000:2.0:1788205212.508650:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788205212.508651:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508653:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205212.508655:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000400:00000200:1.0:1788205212.508656:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:2.0:1788205212.508657:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205212.508660:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1788205212.508661:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1788205212.508661:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508662:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205212.508663:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1788205212.508665:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 02000000:00000001:2.0:1788205212.508665:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205212.508668:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 02000000:00000010:2.0:1788205212.508668:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:2.0:1788205212.508671:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.508672:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:2.0:1788205212.508675:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205212.508676:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205212.508683:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205212.508685:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.508686:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205212.508688:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205212.508689:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205212.508695:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.508696:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205212.508697:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205212.508698:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1788205212.508700:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788205212.508701:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205212.508703:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205212.508704:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508706:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205212.508707:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:2.0:1788205212.508708:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26898 00000100:00000001:1.0:1788205212.508711:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321572613 00000400:00000200:2.0:1788205212.508711:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205212.508714:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205212.508715:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000010:1.0:1788205212.508716:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647c60. 00000100:00000040:2.0:1788205212.508718:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508720:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205212.508721:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205212.508723:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1788205212.508723:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205212.508725:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205212.508726:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:2.0:1788205212.508728:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205212.508729:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321572613 00000020:00000040:2.0:1788205212.508739:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000010:1.0:1788205212.508741:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647420. 00000020:00000001:2.0:1788205212.508741:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:2.0:1788205212.508743:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788205212.508744:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000040:2.0:1788205212.508746:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205212.508747:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205212.508748:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205212.508750:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205212.508750:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205212.508752:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788205212.508752:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205212.508754:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00100000:1.0:1788205212.508756:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205212/real 1788205212] req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205212.508757:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205212.508759:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205212.508761:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205212.508763:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205212.508765:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788205212.508766:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788205212.508766:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000001:1.0:1788205212.508768:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:2.0:1788205212.508768:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000400:00000010:1.0:1788205212.508770:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000008:00000001:2.0:1788205212.508770:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.508772:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000100:00000001:2.0:1788205212.508772:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205212.508774:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000010:2.0:1788205212.508775:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000100:00000001:1.0:1788205212.508777:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1788205212.508777:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205212.508779:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1788205212.508780:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205212.508781:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205212.508782:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205212.508784:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205212.508786:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205212.508789:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788205212.508789:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788205212.508792:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205212.508795:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205212.508802:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205212.508803:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205212.508807:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205212.508814:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.508815:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205212.508817:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205212.508818:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26909) 00000100:00000001:1.0:1788205212.508819:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.508821:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205212.508825:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.508831:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205212.508835:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.508842:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205212.508845:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.508847:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.508848:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.508850:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205212.508852:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205212.508855:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205212.508858:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205212.508861:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205212.508865:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205212.508873:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543524864:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205212.508877:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.508879:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.508882:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74700 x1875055543524864/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.508895:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205212.508897:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205212.508900:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205212.508903:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205212.508905:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205212.508906:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205212.508908:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205212.508910:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067ec00. 02000000:00000001:1.0:1788205212.508915:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205212.508916:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205212.508918:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74700. 00000100:00000001:1.0:1788205212.508921:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205212.508922:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.508924:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.508931:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205212.508932:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.508934:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205212.508936:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205212.508937:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205212.508941:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.508947:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205212.508948:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205212.508950:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205212.508952:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205212.508954:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205212.508957:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524992:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205212.508960:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205212.508962:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205212.508963:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205212.508965:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205212.508967:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205212.508969:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.508972:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205212.508975:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205212.508978:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205212.508981:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543524992, portal 10 00000100:00000001:1.0:1788205212.508983:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205212.508984:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913088 : -109424377638528 : ffff9c7aa6d75180) 00000100:00000040:1.0:1788205212.508990:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.508996:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205212.508998:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205212.508999:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205212.509002:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543524992, offset 0 00000400:00000200:1.0:1788205212.509005:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205212.509009:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205212.509012:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205212.509015:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205212.509017:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205212.509019:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205212.509021:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205212.509024:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471450:471450:256:4294967295] 192.168.201.118@tcp LPNI seq info [471309:471309:8:4294967295] 00000400:00000200:1.0:1788205212.509028:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205212.509033:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205212.509037:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205212.509040:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205212.509043:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205212.509046:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205212.509049:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205212.509052:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205212.509055:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205212.509056:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205212.509059:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205212.509062:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205212.509065:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205212.509066:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205212.509069:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205212.509072:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509127:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.509131:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509136:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.509138:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205212.509141:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205212.509145:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205212.509147:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205212.509149:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321571347 00000100:00000001:1.0:1788205212.509152:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321572613 00000100:00000001:1.0:1788205212.509154:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321571347 00000100:00000001:1.0:1788205212.509155:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321572613 00000100:00000001:1.0:1788205212.509157:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205212.509159:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.509160:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205212.509164:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205212/real 1788205212] req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205212.509170:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205212.509171:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205212.509173:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205212.509174:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205212.509176:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205212.509178:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205212.509181:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205212.509188:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509198:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205212.509199:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509201:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.509202:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205212.509205:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509210:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205212.509213:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509219:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205212.509221:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509222:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.509224:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509225:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205212.509227:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205212.509229:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205212.509232:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205212.509234:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205212.509237:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205212.509244:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543524992:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205212.509247:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205212.509250:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509254:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205212.509255:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205212.509257:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205212.509259:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205212.509261:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205212.509263:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525120:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205212.509266:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205212.509268:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205212.509269:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205212.509271:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205212.509274:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205212.509276:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.509278:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205212.509281:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205212.509283:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205212.509286:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543525120, portal 10 00000100:00000001:1.0:1788205212.509288:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205212.509289:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331922944 : -109424377628672 : ffff9c7aa6d77800) 00000100:00000040:1.0:1788205212.509294:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509299:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205212.509301:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205212.509303:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205212.509305:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543525120, offset 0 00000400:00000200:1.0:1788205212.509308:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205212.509312:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205212.509315:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205212.509318:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205212.509320:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205212.509322:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205212.509324:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205212.509326:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471451:471451:256:4294967295] 192.168.201.118@tcp LPNI seq info [471310:471310:8:4294967295] 00000400:00000200:1.0:1788205212.509330:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205212.509335:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205212.509340:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205212.509343:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205212.509347:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205212.509349:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205212.509352:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205212.509354:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205212.509357:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205212.509359:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205212.509362:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205212.509365:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205212.509367:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205212.509368:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205212.509370:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205212.509373:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509379:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.509381:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509386:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.509387:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205212.509390:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205212.509392:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205212.509395:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205212.509397:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321571348 00000100:00000001:1.0:1788205212.509399:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321572613 00000100:00000001:1.0:1788205212.509401:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321571348 00000100:00000001:1.0:1788205212.509402:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321572613 00000100:00000001:1.0:1788205212.509404:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205212.509406:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.509408:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205212.509411:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205212/real 1788205212] req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205212.509419:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205212.509420:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205212.509422:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205212.509424:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205212.509426:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205212.509428:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205212.509431:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205212.509438:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509444:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205212.509445:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509447:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.509448:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205212.509451:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509456:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205212.509459:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509465:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205212.509466:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509468:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.509469:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509470:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205212.509472:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205212.509475:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205212.509477:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205212.509480:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205212.509484:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205212.509491:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525120:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205212.509494:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205212.509496:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509501:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205212.509503:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205212.509504:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205212.509506:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205212.509508:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205212.509511:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525248:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205212.509513:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205212.509514:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205212.509516:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205212.509517:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205212.509521:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205212.509522:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.509525:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205212.509526:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205212.509529:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205212.509531:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543525248, portal 4 00000100:00000001:1.0:1788205212.509533:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205212.509533:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:1.0:1788205212.509538:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509543:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205212.509545:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205212.509547:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205212.509549:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543525248, offset 0 00000400:00000200:1.0:1788205212.509553:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205212.509557:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205212.509560:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205212.509562:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205212.509564:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205212.509566:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205212.509569:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205212.509571:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471452:471452:256:4294967295] 192.168.201.118@tcp LPNI seq info [471311:471311:8:4294967295] 00000400:00000200:1.0:1788205212.509575:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205212.509580:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205212.509585:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205212.509588:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205212.509591:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205212.509594:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205212.509597:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205212.509600:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205212.509603:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205212.509605:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205212.509608:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205212.509610:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205212.509613:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205212.509614:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205212.509617:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205212.509620:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509626:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.509629:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509633:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.509635:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205212.509637:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205212.509640:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205212.509643:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205212.509645:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321571357 00000100:00000001:1.0:1788205212.509647:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321572613 00000100:00000001:1.0:1788205212.509649:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321571357 00000100:00000001:1.0:1788205212.509651:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321572613 00000100:00000001:1.0:1788205212.509653:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205212.509654:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.509656:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205212.509659:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205212/real 1788205212] req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205212.509665:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205212.509666:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205212.509667:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205212.509699:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205212.509701:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205212.509703:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205212.509707:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205212.509714:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509719:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205212.509720:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509722:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.509723:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205212.509726:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509732:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205212.509735:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509740:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205212.509743:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509744:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.509745:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.509747:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205212.509749:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205212.509751:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205212.509754:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205212.509757:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205212.509760:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205212.509767:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525248:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205212.509770:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205212.509773:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509779:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205212.509780:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205212.509782:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205212.509784:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205212.509786:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205212.509788:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525376:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205212.509791:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205212.509793:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205212.509794:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205212.509796:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205212.509799:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205212.509801:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205212.509804:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205212.509806:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205212.509808:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205212.509811:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543525376, portal 4 00000100:00000001:1.0:1788205212.509814:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205212.509815:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:1.0:1788205212.509836:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509842:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205212.509844:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205212.509846:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205212.509848:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543525376, offset 0 00000400:00000200:1.0:1788205212.509851:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205212.509855:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205212.509858:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205212.509860:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205212.509862:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205212.509864:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205212.509866:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205212.509869:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471453:471453:256:4294967295] 192.168.201.118@tcp LPNI seq info [471312:471312:8:4294967295] 00000400:00000200:1.0:1788205212.509873:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205212.509879:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205212.509883:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205212.509885:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205212.509889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205212.509891:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1788205212.509895:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205212.509897:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1788205212.509901:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205212.509902:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205212.509905:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205212.509908:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205212.509911:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205212.509912:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205212.509914:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205212.509917:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509928:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.509931:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205237 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.509937:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.509938:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205212.509940:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205212.509943:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205212.509945:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205212.509947:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321571380 00000100:00000001:1.0:1788205212.509949:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321572613 00000100:00000001:1.0:1788205212.509951:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321571380 00000100:00000001:1.0:1788205212.509953:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321572613 00000100:00000001:1.0:1788205212.509954:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205212.509956:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.509957:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205212.509994:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205212/real 1788205212] req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205212.510001:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205212.510002:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205212.510004:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205212.510006:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205212.510007:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205212.510009:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205212.510012:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205212.510019:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.510024:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205212.510026:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510027:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.510028:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205212.510031:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.510037:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205212.510040:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.510045:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205212.510047:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510048:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205212.510049:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510051:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205212.510052:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205212.510054:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205212.510057:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205212.510060:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205212.510063:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205212.510070:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525376:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205212.510072:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510074:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.510077:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75180 x1875055543524992/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.510082:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205212.510084:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205212.510085:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:1.0:1788205212.510088:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205212.510125:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205212.510128:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205212.510130:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205212.510132:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788205212.510135:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205212.510136:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205212.510137:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75180. 00000100:00000001:1.0:1788205212.510139:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205212.510141:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510142:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.510145:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77800 x1875055543525120/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.510151:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205212.510152:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205212.510153:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205212.510155:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205212.510156:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205212.510157:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205212.510159:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205212.510160:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:1.0:1788205212.510162:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205212.510163:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205212.510164:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77800. 00000100:00000001:1.0:1788205212.510166:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205212.510168:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510169:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.510172:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543525248/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.510176:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205212.510177:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205212.510178:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205212.510181:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205212.510182:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205212.510183:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205212.510184:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205212.510186:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:1.0:1788205212.510187:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205212.510189:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205212.510190:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788205212.510191:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205212.510192:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510194:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205212.510196:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77b80 x1875055543525376/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205237 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205212.510201:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205212.510202:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205212.510203:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205212.510205:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205212.510206:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205212.510207:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205212.510209:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205212.510210:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d000. 02000000:00000001:1.0:1788205212.510211:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205212.510212:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205212.510213:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77b80. 00000100:00000001:1.0:1788205212.510215:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205212.510215:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510217:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205212.510220:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205212.510221:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205212.510223:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205212.510224:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205213.340263:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205213.340272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205213.340274:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205213.340278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205213.340280:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205213.340286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205213.340289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205213.340293:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205213.340296:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205213.340300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205213.340303:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205213.340307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205213.340310:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205213.340311:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205213.340316:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205213.340325:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205213.340330:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205213.340457:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205213.340463:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205213.340467:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205213.340924:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205213.340930:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205213.340931:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205213.340935:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205213.340939:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205213.340942:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205213.340945:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205213.340948:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205213.340950:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205213.852258:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321572613, at 4321572672 00000100:00000040:2.0:1788205213.852259:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321572613, at 4321572672 00000100:00000010:3.0:1788205213.852278:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647420. 00000100:00000010:2.0:1788205213.852279:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00000100:00000001:2.0:1788205214.364231:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205214.364232:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205214.364242:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205214.364243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205214.364246:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205214.364247:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205214.364256:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205214.364257:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205214.364265:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205214.364265:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205214.364273:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205214.364274:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205214.364284:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205214.364285:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205214.364289:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205214.364290:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205214.364296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205214.364297:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205214.364424:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205214.364429:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205214.365603:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205214.365606:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205214.365610:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205214.365611:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205214.365613:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205214.365614:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205214.365662:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205214.365666:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205214.365668:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205215.388148:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205215.388148:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205215.388155:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205215.388155:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205215.388157:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205215.388158:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205215.388158:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388161:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388166:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205215.388166:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205215.388166:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205215.388170:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205215.388171:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205215.388171:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205215.388173:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205215.388176:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205215.388176:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205215.388180:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205215.388183:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205215.388185:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205215.388186:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205215.388188:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205215.388190:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388192:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388196:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205215.388198:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388200:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388201:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388204:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205215.388205:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205215.388206:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205215.388207:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388208:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205215.388209:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388210:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388211:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388214:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205215.388215:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205215.388216:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205215.388217:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388218:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205215.388219:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388219:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205215.388220:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205215.388293:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205215.388300:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205215.388303:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205216.412168:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205216.412182:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205216.412184:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205216.412189:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205216.412193:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205216.412197:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205216.412200:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205216.412202:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205216.412211:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205216.412380:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205216.412384:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205216.412386:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205216.412389:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205216.412391:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205216.412393:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205216.412395:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205216.412397:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205216.412399:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205216.413649:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205216.413653:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205216.413658:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205216.413659:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205216.413662:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205216.413664:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205216.413667:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205216.413668:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205216.413864:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205216.413868:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205216.413871:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205217.436250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205217.436257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205217.436263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205217.436264:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205217.436267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205217.436268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205217.436272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205217.436273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205217.436360:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205217.436364:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205217.436367:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205217.436710:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205217.436715:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205217.436717:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205217.436721:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205217.436725:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205217.436728:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205217.436731:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205217.436748:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205217.436751:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205217.436936:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205217.436940:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205217.436942:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205217.436945:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205217.436948:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205217.436951:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205217.436953:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205217.436955:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205217.436958:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205217.628207:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205217.628217:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788205217.628220:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205217.628224:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205217.628228:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205217.628235:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26904 00000400:00000200:2.0:1788205217.628246:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205217.628253:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205217.628257:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205217.628260:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628263:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205217.628266:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628268:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205217.628272:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205217.628274:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:2.0:1788205217.628300:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205217.628304:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:2.0:1788205217.628307:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:2.0:1788205217.628309:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:2.0:1788205217.628313:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628315:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205217.628317:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628319:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205217.628322:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:2.0:1788205217.628325:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628328:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205217.628331:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205217.628335:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205217.628337:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.628340:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:2.0:1788205217.628342:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205217.628352:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74a80. 00000020:00000040:2.0:1788205217.628354:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:2.0:1788205217.628358:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205217.628359:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.628361:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205217.628363:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205217.628366:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205217.628372:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e800. 00000100:00000001:2.0:1788205217.628377:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205217.628383:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205217.628390:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205217.628394:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205217.628396:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000427006b7] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205217.628415:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205217.628417:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205217.628420:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788205217.628421:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205217.628422:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788205217.628424:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205217.628425:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26904 00000400:00000200:2.0:1788205217.628428:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205217.628430:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205217.628432:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205217.628434:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628436:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205217.628437:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628438:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205217.628440:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205217.628441:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:2.0:1788205217.628442:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205217.628444:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:2.0:1788205217.628445:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:2.0:1788205217.628446:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:2.0:1788205217.628448:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628449:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205217.628451:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628452:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205217.628454:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:2.0:1788205217.628455:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205217.628457:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205217.628457:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205217.628459:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205217.628463:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205217.628464:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.628466:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205217.628469:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788205217.628472:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.628473:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205217.628473:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.628478:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000010:2.0:1788205217.628478:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74380. 00000020:00000040:2.0:1788205217.628482:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:2.0:1788205217.628484:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205217.628485:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.628487:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205217.628488:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1788205217.628489:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788205217.628490:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1788205217.628496:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067d000. 00000100:00000001:2.0:1788205217.628500:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.628506:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1788205217.628506:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205217.628510:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.628516:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788205217.628518:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788205217.628519:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.628523:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1788205217.628524:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004198c78f] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205217.628531:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.628533:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205217.628536:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788205217.628539:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543525504:192.168.201.118@tcp:250:kworker.0 00000100:00080000:2.0:1788205217.628540:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205217.628546:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205217.628550:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205217.628553:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.628555:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205217.628560:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205217.628568:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205217.628574:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205217.628589:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205217.628598:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205217.628605:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543525504, portal 25 00000100:00000001:1.0:1788205217.628608:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205217.628609:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331911296 : -109424377640320 : ffff9c7aa6d74a80) 00000100:00000040:1.0:1788205217.628615:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.628622:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205217.628625:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205217.628628:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205217.628632:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543525504, offset 0 00000400:00000200:1.0:1788205217.628638:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205217.628645:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.628651:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205217.628657:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205217.628660:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205217.628663:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205217.628666:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205217.628670:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471454:471454:256:4294967295] 192.168.201.118@tcp LPNI seq info [471313:471313:8:4294967295] 00000400:00000200:1.0:1788205217.628674:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.628684:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205217.628689:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.628694:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205217.628709:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205217.628716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205217.628722:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205217.628725:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205217.628729:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.628732:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205217.628738:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1788205217.628741:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205217.628745:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1788205217.628746:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1788205217.628752:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1788205217.628755:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00080000:2.0:1788205217.628757:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26904 00000400:00000010:1.0:1788205217.628760:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000400:00000200:2.0:1788205217.628765:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205217.628766:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1788205217.628771:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000200:1.0:1788205217.628774:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205217.628775:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205217.628781:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628787:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205217.628791:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628794:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205217.628797:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:2.0:1788205217.628798:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:2.0:1788205217.628800:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205217.628802:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:2.0:1788205217.628804:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788205217.628805:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788205217.628806:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:2.0:1788205217.628812:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:1.0:1788205217.628813:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205217.628815:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205217.628820:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628826:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205217.628831:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1788205217.628832:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00000001:1.0:1788205217.628836:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1788205217.628838:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205217.628842:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.628844:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205217.628845:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205217.628850:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205217.628852:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:2.0:1788205217.628855:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.628857:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321577733 00000100:00000001:2.0:1788205217.628859:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205217.628864:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000010:1.0:1788205217.628867:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647180. 00000008:00000001:2.0:1788205217.628868:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.628873:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.628878:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000010:2.0:1788205217.628878:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d75500. 00000020:00000040:2.0:1788205217.628883:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000100:00000001:1.0:1788205217.628887:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321577733 02000000:00000001:2.0:1788205217.628888:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205217.628890:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.628893:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:1.0:1788205217.628894:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647000. 02000000:00000001:2.0:1788205217.628895:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205217.628899:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.628900:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000010:2.0:1788205217.628903:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067fc00. 00000100:00000001:1.0:1788205217.628905:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788205217.628907:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.628909:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205217.628913:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205217.628914:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788205217.628922:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1788205217.628923:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205217/real 1788205217] req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205217.628925:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205217.628931:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000679714e] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205217.628947:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205217.628949:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205217.628951:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205217.628957:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205217.628960:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:2.0:1788205217.628960:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205217.628964:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205217.628965:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1788205217.628967:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788205217.628969:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205217.628970:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1788205217.628973:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205217.628973:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1788205217.628978:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26904 00000400:00000200:2.0:1788205217.628985:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205217.628989:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000200:1.0:1788205217.628996:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788205217.628997:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1788205217.629005:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205217.629008:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788205217.629009:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205217.629011:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205217.629013:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205217.629016:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.629030:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205217.629031:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:2.0:1788205217.629035:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00100000:1.0:1788205217.629037:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1788205217.629038:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:2.0:1788205217.629041:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:2.0:1788205217.629044:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:2.0:1788205217.629048:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:2.0:1788205217.629054:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205217.629055:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1788205217.629060:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.629062:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205217.629065:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1788205217.629066:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.629069:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205217.629073:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:2.0:1788205217.629077:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205217.629081:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205217.629084:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205217.629087:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205217.629089:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788205217.629108:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.629110:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205217.629111:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1788205217.629113:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.629114:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205217.629118:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d77b80. 00000020:00000040:2.0:1788205217.629121:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205217.629123:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788205217.629125:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.629127:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788205217.629129:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205217.629133:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788205217.629133:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629139:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.629141:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1788205217.629141:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067e000. 00000100:00000001:1.0:1788205217.629147:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788205217.629148:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629150:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205217.629156:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788205217.629157:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205217.629169:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205217.629171:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205217.629172:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000040:2.0:1788205217.629176:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000037de4ed3] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629178:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205217.629187:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205217.629191:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788205217.629196:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788205217.629204:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1788205217.629208:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543525504:192.168.201.118@tcp:250:kworker.0 00000100:00000001:2.0:1788205217.629208:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788205217.629211:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205217.629217:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205217.629218:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205217.629222:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788205217.629224:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26904 00000100:00000040:1.0:1788205217.629229:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74a80 x1875055543525504/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205217.629233:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205217.629240:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:2.0:1788205217.629245:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205217.629246:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00000040:2.0:1788205217.629250:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205217.629251:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788205217.629255:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1788205217.629256:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 00000100:00000040:2.0:1788205217.629260:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205217.629262:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1788205217.629264:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205217.629265:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000001:2.0:1788205217.629269:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205217.629270:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000040:2.0:1788205217.629272:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205217.629274:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 00000020:00000001:2.0:1788205217.629277:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 02000000:00000010:1.0:1788205217.629279:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e800. 00000020:00000040:2.0:1788205217.629282:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 02000000:00000001:1.0:1788205217.629286:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1788205217.629286:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 02000000:00000001:1.0:1788205217.629289:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:2.0:1788205217.629291:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000010:1.0:1788205217.629293:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74a80. 00000100:00000040:2.0:1788205217.629297:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205217.629299:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205217.629302:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205217.629302:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1788205217.629306:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:2.0:1788205217.629309:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:2.0:1788205217.629312:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:1.0:1788205217.629316:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788205217.629318:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:2.0:1788205217.629323:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205217.629325:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788205217.629327:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788205217.629329:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.629331:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:2.0:1788205217.629333:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1788205217.629335:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.629336:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1788205217.629339:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788205217.629341:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:2.0:1788205217.629341:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:2.0:1788205217.629344:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205217.629346:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788205217.629346:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205217.629351:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788205217.629353:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788205217.629353:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788205217.629355:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629357:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1788205217.629359:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b9067c800. 00000100:00000001:1.0:1788205217.629360:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788205217.629363:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205217.629368:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788205217.629370:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788205217.629387:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788205217.629390:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629392:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205217.629395:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205217.629397:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6f41207] to pc [ptlrpcd_rcv+-1] req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205217.629399:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205217.629402:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205217.629407:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205217.629414:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525632:192.168.201.118@tcp:38:kworker.0 00000100:00000001:2.0:1788205217.629415:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629422:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1788205217.629423:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26914) 02000000:00000001:1.0:1788205217.629426:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205217.629428:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.629431:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205217.629434:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205217.629437:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205217.629441:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205217.629444:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205217.629448:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205217.629451:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543525632, portal 10 00000100:00000001:1.0:1788205217.629453:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205217.629455:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331909504 : -109424377642112 : ffff9c7aa6d74380) 00000100:00000040:1.0:1788205217.629462:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629468:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205217.629470:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205217.629472:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205217.629474:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543525632, offset 0 00000400:00000200:1.0:1788205217.629478:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205217.629482:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.629486:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205217.629489:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205217.629491:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205217.629494:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205217.629496:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205217.629500:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471455:471455:256:4294967295] 192.168.201.118@tcp LPNI seq info [471314:471314:8:4294967295] 00000400:00000200:1.0:1788205217.629504:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.629510:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205217.629515:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.629518:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205217.629522:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205217.629525:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205217.629530:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205217.629532:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205217.629536:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.629537:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205217.629540:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205217.629543:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205217.629546:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205217.629547:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205217.629550:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205217.629553:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629559:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205217.629562:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629568:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629569:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205217.629572:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205217.629576:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205217.629578:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205217.629580:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321576467 00000100:00000001:1.0:1788205217.629583:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321577733 00000100:00000001:1.0:1788205217.629584:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321576467 00000100:00000001:1.0:1788205217.629586:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321577733 00000100:00000001:1.0:1788205217.629589:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205217.629591:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629592:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205217.629596:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205217/real 1788205217] req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205217.629603:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205217.629604:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205217.629606:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205217.629608:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205217.629610:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205217.629612:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205217.629615:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205217.629623:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629630:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205217.629632:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.629633:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.629634:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205217.629638:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629644:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205217.629648:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629653:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205217.629655:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.629657:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.629658:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.629659:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205217.629661:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205217.629664:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205217.629666:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205217.629668:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205217.629672:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205217.629679:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525632:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205217.629683:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205217.629686:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629692:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205217.629693:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.629695:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205217.629697:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205217.629699:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205217.629701:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525760:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205217.629704:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205217.629706:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205217.629707:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.629709:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205217.629713:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205217.629715:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205217.629717:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205217.629720:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205217.629723:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205217.629725:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543525760, portal 10 00000100:00000001:1.0:1788205217.629727:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205217.629728:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331913984 : -109424377637632 : ffff9c7aa6d75500) 00000100:00000040:1.0:1788205217.629733:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629743:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205217.629745:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205217.629747:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205217.629749:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543525760, offset 0 00000400:00000200:1.0:1788205217.629753:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205217.629756:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.629759:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205217.629762:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205217.629765:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205217.629767:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205217.629769:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205217.629772:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471456:471456:256:4294967295] 192.168.201.118@tcp LPNI seq info [471315:471315:8:4294967295] 00000400:00000200:1.0:1788205217.629775:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.629781:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205217.629785:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.629788:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205217.629792:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205217.629794:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205217.629798:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205217.629801:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205217.629805:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.629806:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205217.629809:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205217.629812:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205217.629814:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205217.629816:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205217.629818:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205217.629822:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629829:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205217.629832:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629843:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629845:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205217.629848:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205217.629850:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205217.629854:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205217.629855:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321576468 00000100:00000001:1.0:1788205217.629858:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321577733 00000100:00000001:1.0:1788205217.629860:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321576468 00000100:00000001:1.0:1788205217.629862:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321577733 00000100:00000001:1.0:1788205217.629864:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205217.629866:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.629867:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205217.629871:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205217/real 1788205217] req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205217.629879:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205217.629880:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205217.629882:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205217.629884:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205217.629885:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205217.629887:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205217.629890:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205217.629897:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629903:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205217.629904:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.629905:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.629906:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205217.629909:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629916:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205217.629920:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629926:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205217.629928:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.629929:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.629930:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.629932:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205217.629935:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205217.629937:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205217.629940:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205217.629942:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205217.629946:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205217.629953:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525760:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205217.629957:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205217.629959:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629964:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205217.629965:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.629966:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205217.629967:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205217.629968:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205217.629970:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525888:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205217.629972:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205217.629973:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205217.629974:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.629976:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205217.629978:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205217.629979:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205217.629981:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205217.629983:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205217.629985:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205217.629987:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543525888, portal 4 00000100:00000001:1.0:1788205217.629988:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205217.629989:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331923840 : -109424377627776 : ffff9c7aa6d77b80) 00000100:00000040:1.0:1788205217.629992:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.629997:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205217.629999:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205217.630001:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205217.630003:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543525888, offset 0 00000400:00000200:1.0:1788205217.630006:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205217.630010:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.630013:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205217.630015:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205217.630030:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205217.630033:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205217.630035:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205217.630038:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471457:471457:256:4294967295] 192.168.201.118@tcp LPNI seq info [471316:471316:8:4294967295] 00000400:00000200:1.0:1788205217.630042:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.630047:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205217.630052:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.630055:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205217.630059:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205217.630061:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205217.630065:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205217.630068:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205217.630071:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.630074:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205217.630077:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205217.630080:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205217.630111:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205217.630113:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205217.630115:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205217.630118:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630123:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205217.630125:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630130:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.630132:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205217.630134:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205217.630137:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205217.630140:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205217.630141:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321576478 00000100:00000001:1.0:1788205217.630144:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321577733 00000100:00000001:1.0:1788205217.630150:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321576478 00000100:00000001:1.0:1788205217.630152:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321577733 00000100:00000001:1.0:1788205217.630154:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205217.630156:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.630158:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205217.630162:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205217/real 1788205217] req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205217.630169:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205217.630170:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205217.630172:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205217.630175:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205217.630176:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205217.630178:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205217.630181:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205217.630189:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630195:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205217.630196:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630198:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.630200:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205217.630203:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630210:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205217.630213:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630220:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205217.630222:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630224:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.630225:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630226:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205217.630227:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205217.630229:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205217.630231:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205217.630233:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205217.630235:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205217.630240:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543525888:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205217.630242:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630243:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205217.630245:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74380 x1875055543525632/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630249:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205217.630250:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205217.630252:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:1.0:1788205217.630253:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205217.630254:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205217.630255:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205217.630257:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205217.630258:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067d000. 02000000:00000001:1.0:1788205217.630260:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205217.630261:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205217.630262:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74380. 00000100:00000001:1.0:1788205217.630264:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205217.630265:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630266:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205217.630268:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d75500 x1875055543525760/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630271:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205217.630272:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205217.630273:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:1.0:1788205217.630275:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205217.630275:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205217.630276:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205217.630277:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205217.630277:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067fc00. 02000000:00000001:1.0:1788205217.630279:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205217.630279:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205217.630280:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d75500. 00000100:00000001:1.0:1788205217.630282:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205217.630282:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630283:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205217.630285:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d77b80 x1875055543525888/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630288:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205217.630289:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205217.630290:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205217.630291:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205217.630292:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205217.630292:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205217.630293:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205217.630294:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067e000. 02000000:00000001:1.0:1788205217.630295:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205217.630296:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205217.630297:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d77b80. 00000100:00000001:1.0:1788205217.630298:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205217.630299:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630300:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630302:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205217.630303:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.630304:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205217.630305:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205217.630306:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205217.630308:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630313:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205217.630314:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.630316:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205217.630318:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205217.630319:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205217.630322:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526016:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205217.630325:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205217.630327:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205217.630328:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205217.630330:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205217.630332:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205217.630334:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205217.630337:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205217.630340:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205217.630343:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205217.630346:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543526016, portal 4 00000100:00000001:1.0:1788205217.630348:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205217.630350:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649331908608 : -109424377643008 : ffff9c7aa6d74000) 00000100:00000040:1.0:1788205217.630355:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630359:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205217.630360:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205217.630362:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205217.630363:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543526016, offset 0 00000400:00000200:1.0:1788205217.630366:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205217.630368:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.630371:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205217.630373:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205217.630374:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205217.630376:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205217.630377:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205217.630379:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471458:471458:256:4294967295] 192.168.201.118@tcp LPNI seq info [471317:471317:8:4294967295] 00000400:00000200:1.0:1788205217.630381:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205217.630386:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205217.630390:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.630393:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205217.630397:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205217.630399:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1788205217.630403:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205217.630405:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1788205217.630409:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205217.630411:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205217.630413:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205217.630417:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205217.630420:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205217.630422:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205217.630424:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205217.630427:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630437:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205217.630439:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205242 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630443:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.630444:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205217.630446:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205217.630448:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205217.630450:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205217.630452:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321576501 00000100:00000001:1.0:1788205217.630455:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321577733 00000100:00000001:1.0:1788205217.630457:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321576501 00000100:00000001:1.0:1788205217.630458:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321577733 00000100:00000001:1.0:1788205217.630460:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205217.630462:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.630464:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205217.630467:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205217/real 1788205217] req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205217.630474:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205217.630476:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205217.630477:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205217.630480:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205217.630481:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205217.630483:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205217.630486:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205217.630494:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630500:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205217.630502:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630504:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.630505:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205217.630508:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630514:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205217.630518:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630524:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205217.630527:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630528:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205217.630530:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630532:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205217.630533:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205217.630536:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205217.630539:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205217.630541:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205217.630545:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205217.630553:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aa6d74000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526016:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205217.630557:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630558:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205217.630561:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aa6d74000 x1875055543526016/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205242 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205217.630567:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205217.630569:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205217.630571:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:1.0:1788205217.630573:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205217.630574:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205217.630576:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205217.630577:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205217.630579:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b9067c800. 02000000:00000001:1.0:1788205217.630581:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205217.630582:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205217.630584:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aa6d74000. 00000100:00000001:1.0:1788205217.630586:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205217.630587:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630589:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205217.630592:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205217.630593:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205217.630595:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205217.630596:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205218.460242:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205218.460272:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205218.460274:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205218.460280:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205218.460292:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205218.460296:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205218.460299:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205218.460311:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205218.460314:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205218.460440:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205218.460444:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205218.460445:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205218.460448:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205218.460451:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205218.460453:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205218.460455:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205218.460457:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205218.460459:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205218.460561:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205218.460574:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205218.460579:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205218.460580:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205218.460584:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205218.460585:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205218.460599:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205218.460600:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205218.460687:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205218.460691:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205218.460694:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205218.972390:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321577733, at 4321577792 00000100:00000010:3.0:1788205218.972407:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647000. 00000100:00000040:2.0:1788205218.972505:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321577733, at 4321577792 00000100:00000010:2.0:1788205218.972512:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00000100:00000001:3.0:1788205219.484190:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205219.484197:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205219.484198:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205219.484199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205219.484205:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205219.484205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205219.484209:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205219.484210:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205219.484214:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205219.484215:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205219.484217:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205219.484220:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205219.484222:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205219.484232:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205219.484235:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205219.484238:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205219.484240:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205219.484242:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205219.484385:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205219.484390:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205219.484416:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205219.484420:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205219.484424:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205219.484426:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205219.484428:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205219.484430:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205219.484434:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205219.484435:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205219.484438:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205219.484439:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205219.484443:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205219.484444:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205219.484481:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205219.484484:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205219.484486:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205220.508215:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205220.508218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205220.508224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205220.508226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205220.508230:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205220.508232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205220.508235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205220.508236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205220.508240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205220.508242:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205220.508243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205220.508248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205220.508249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205220.508253:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205220.508253:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205220.508258:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205220.508261:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205220.508335:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205220.508339:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205220.508343:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205220.508357:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205220.508362:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205220.508364:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205220.508367:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205220.508369:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205220.508371:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205220.508374:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205220.508376:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205220.508377:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205221.532317:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205221.532325:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205221.532327:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205221.532333:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205221.532337:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205221.532340:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205221.532343:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205221.532346:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205221.532349:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205221.532422:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205221.532425:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205221.532428:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205221.532429:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205221.532431:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205221.532432:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205221.532434:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205221.532435:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205221.532566:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205221.532574:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205221.532579:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205221.533027:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205221.533032:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205221.533034:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205221.533037:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205221.533040:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205221.533042:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205221.533045:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205221.533046:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205221.533048:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205222.556256:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205222.556266:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205222.556268:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205222.556275:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205222.556279:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205222.556283:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205222.556286:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205222.556289:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205222.556292:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205222.556582:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205222.556588:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205222.556595:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205222.556596:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205222.556600:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205222.556601:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205222.556605:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205222.556606:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205222.558103:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205222.558109:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205222.558115:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205222.558169:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205222.558174:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205222.558176:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205222.558178:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205222.558182:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205222.558185:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205222.558187:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205222.558190:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205222.558192:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205222.748406:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205222.748420:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205222.748424:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205222.748428:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205222.748433:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205222.748443:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26909 00000400:00000200:1.0:1788205222.748457:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205222.748465:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205222.748469:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205222.748473:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.748476:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.748478:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.748481:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205222.749768:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205222.749774:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205222.749778:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205222.749782:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205222.749785:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205222.749788:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205222.749795:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.749798:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.749800:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.749801:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205222.749803:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205222.749806:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205222.749808:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205222.749810:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205222.749814:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205222.749816:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.749817:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205222.749820:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205222.749829:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000020:00000040:1.0:1788205222.749832:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205222.749835:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205222.749836:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.749838:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205222.749840:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205222.749843:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205222.749850:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000001:1.0:1788205222.749856:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205222.749862:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205222.749870:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205222.749873:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205222.749876:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205222.749895:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205222.749898:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205222.749901:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205222.749909:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205222.749911:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205222.749913:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205222.749915:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26909 00000400:00000200:1.0:1788205222.749922:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205222.749927:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205222.749940:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205222.749942:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.749943:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.749945:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.749946:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205222.749948:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205222.749948:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205222.749949:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205222.749951:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205222.749952:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205222.749953:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205222.749955:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.749957:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.749958:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.749959:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205222.749960:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205222.749962:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205222.749963:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205222.749964:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205222.749968:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205222.749969:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.749971:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205222.749974:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205222.749975:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.749977:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205222.749979:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205222.749980:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205222.749982:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205222.749983:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.749984:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205222.749985:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205222.749986:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205222.749988:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb400. 00000100:00000001:1.0:1788205222.749991:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205222.749994:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205222.749998:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205222.750000:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205222.750002:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205222.750007:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205222.750009:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205222.750011:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205222.750012:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205222.750013:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205222.750015:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205222.750016:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26909 00000400:00000200:1.0:1788205222.750019:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205222.750021:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205222.750022:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205222.750024:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750025:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.750026:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750027:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205222.750029:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205222.750029:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205222.750030:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205222.750031:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205222.750032:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205222.750033:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205222.750035:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750036:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.750037:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750038:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205222.750039:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205222.750041:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750043:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205222.750044:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205222.750046:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205222.750048:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750050:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205222.750051:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205222.750053:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750055:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205222.750060:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000020:00000040:1.0:1788205222.750062:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205222.750065:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205222.750066:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750068:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205222.750069:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205222.750071:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205222.750074:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba800. 00000100:00000001:1.0:1788205222.750078:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205222.750082:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205222.750089:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205222.750107:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205222.750111:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205222.750119:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205222.750122:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205222.750126:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205222.750128:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205222.750129:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205222.750132:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205222.750137:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26909 00000400:00000200:1.0:1788205222.750141:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205222.750148:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205222.750150:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205222.750153:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750155:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.750156:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750158:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205222.750161:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:3.0:1788205222.750162:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1788205222.750164:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205222.750167:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205222.750169:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205222.750171:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205222.750173:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205222.750175:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750177:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.750178:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750180:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205222.750181:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205222.750184:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750185:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205222.750187:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205222.750189:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205222.750191:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750193:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205222.750194:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205222.750196:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750198:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205222.750201:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205222.750203:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205222.750205:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205222.750206:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750208:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205222.750209:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205222.750211:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205222.750213:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9400. 00000100:00000001:1.0:1788205222.750216:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205222.750220:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205222.750226:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205222.750228:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205222.750231:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205222.750238:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205222.750240:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205222.750242:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205222.750243:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205222.750245:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205222.750247:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205222.750249:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26909 00000400:00000200:1.0:1788205222.750251:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205222.750253:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205222.750255:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205222.750257:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750259:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.750260:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750262:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205222.750263:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205222.750264:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205222.750265:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205222.750267:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205222.750269:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205222.750270:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205222.750273:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750274:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205222.750275:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750277:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205222.750279:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205222.750281:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205222.750282:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205222.750284:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205222.750286:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205222.750288:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750289:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205222.750291:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205222.750293:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750294:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205222.750297:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205222.750299:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205222.750300:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205222.750301:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205222.750303:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205222.750304:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205222.750306:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205222.750309:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000001:1.0:1788205222.750312:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205222.750316:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205222.750322:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205222.750323:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205222.750325:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205222.750330:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205222.750334:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26919) 00000100:00000001:3.0:1788205222.750493:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205222.750498:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205222.750510:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.750521:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205222.750523:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.750528:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205222.750532:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205222.750535:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205222.750538:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543526144:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205222.750542:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205222.750548:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205222.750551:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.750553:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205222.750563:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205222.750565:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205222.751875:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205222.751886:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205222.751911:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205222.751918:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543526144, portal 25 00000100:00000001:3.0:1788205222.751921:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205222.751922:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788205222.751954:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.751964:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205222.751966:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205222.751969:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205222.751972:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543526144, offset 0 00000400:00000200:3.0:1788205222.751978:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205222.751987:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.751994:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205222.752000:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205222.752005:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205222.752009:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205222.752013:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205222.752019:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471459:471459:256:4294967295] 192.168.201.118@tcp LPNI seq info [471318:471318:8:4294967295] 00000400:00000200:3.0:1788205222.752025:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.752035:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205222.752053:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.752074:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205222.752084:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205222.752104:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205222.752112:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205222.752116:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205222.752121:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.752125:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205222.752133:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205222.752138:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205222.752147:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205222.752151:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205222.752156:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205222.752161:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752177:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.752180:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752187:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.752188:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205222.752198:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205222.752204:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205222.752207:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321582857 00000100:00000010:3.0:1788205222.752214:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc300. 00000100:00000001:3.0:1788205222.752223:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205222.752231:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321582857 00000100:00000010:3.0:1788205222.752234:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcf60. 00000100:00000001:3.0:1788205222.752237:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205222.752240:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205222.752243:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.752248:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205222.752252:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205222/real 1788205222] req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205222.752262:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205222.752264:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205222.752267:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205222.752269:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205222.752271:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205222.752274:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205222.752278:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205222.752286:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752293:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205222.752295:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752297:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.752299:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205222.752303:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752310:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205222.752315:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752323:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205222.752326:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752327:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.752328:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752332:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205222.752334:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205222.752339:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205222.752342:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205222.752344:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205222.752348:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205222.752356:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543526144:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205222.752362:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205222.752366:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752373:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205222.752375:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.752378:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205222.752381:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205222.752384:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205222.752387:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526272:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205222.752391:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205222.752395:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205222.752398:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.752400:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205222.752405:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205222.752407:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205222.752411:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205222.752413:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205222.752416:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205222.752419:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543526272, portal 10 00000100:00000001:3.0:1788205222.752421:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205222.752422:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205222.752429:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752436:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205222.752439:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205222.752440:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205222.752442:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543526272, offset 0 00000400:00000200:3.0:1788205222.752445:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205222.752449:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.752453:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205222.752455:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205222.752458:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205222.752460:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205222.752462:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205222.752465:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471460:471460:256:4294967295] 192.168.201.118@tcp LPNI seq info [471319:471319:8:4294967295] 00000400:00000200:3.0:1788205222.752469:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.752476:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205222.752480:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.752482:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205222.752486:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205222.752489:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205222.752493:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205222.752496:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205222.752499:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.752500:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205222.752503:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205222.752507:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205222.752510:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205222.752512:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205222.752514:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205222.752517:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752524:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.752527:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752539:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.752540:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205222.752543:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205222.752547:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205222.752551:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205222.752552:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321581590 00000100:00000001:3.0:1788205222.752555:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321582857 00000100:00000001:3.0:1788205222.752557:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321581590 00000100:00000001:3.0:1788205222.752559:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321582857 00000100:00000001:3.0:1788205222.752561:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205222.752563:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.752565:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205222.752569:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205222/real 1788205222] req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205222.752576:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205222.752578:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205222.752579:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205222.752581:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205222.752583:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205222.752585:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205222.752588:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205222.752595:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752602:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205222.752603:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752605:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.752606:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205222.752609:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752616:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205222.752620:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752627:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205222.752629:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752630:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.752631:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752633:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205222.752636:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205222.752638:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205222.752642:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205222.752644:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205222.752648:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205222.752655:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526272:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205222.752658:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205222.752661:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752668:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205222.752669:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.752671:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205222.752673:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205222.752675:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205222.752677:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526400:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205222.752680:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205222.752682:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205222.752684:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.752686:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205222.752689:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205222.752691:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205222.752694:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205222.752697:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205222.752699:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205222.752702:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543526400, portal 10 00000100:00000001:3.0:1788205222.752703:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205222.752705:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205222.752710:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752716:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205222.752718:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205222.752720:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205222.752722:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543526400, offset 0 00000400:00000200:3.0:1788205222.752725:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205222.752728:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.752732:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205222.752734:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205222.752737:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205222.752739:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205222.752742:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205222.752745:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471461:471461:256:4294967295] 192.168.201.118@tcp LPNI seq info [471320:471320:8:4294967295] 00000400:00000200:3.0:1788205222.752749:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.752754:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205222.752759:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.752761:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205222.752765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205222.752768:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205222.752771:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205222.752773:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205222.752777:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.752793:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205222.752796:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205222.752799:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205222.752802:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205222.752803:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205222.752805:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205222.752808:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752816:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.752819:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752826:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.752828:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205222.752830:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205222.752832:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205222.752835:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205222.752837:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321581591 00000100:00000001:3.0:1788205222.752839:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321582857 00000100:00000001:3.0:1788205222.752841:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321581591 00000100:00000001:3.0:1788205222.752843:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321582857 00000100:00000001:3.0:1788205222.752845:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205222.752847:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.752849:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205222.752852:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205222/real 1788205222] req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205222.752859:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205222.752860:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205222.752862:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205222.752864:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205222.752865:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205222.752867:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205222.752883:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205222.752891:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752896:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205222.752898:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752899:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.752901:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205222.752904:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752910:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205222.752914:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752920:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205222.752923:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752924:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.752925:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.752927:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205222.752939:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205222.752941:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205222.752945:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205222.752947:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205222.752950:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205222.752958:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526400:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205222.752962:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205222.752966:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.752972:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205222.752973:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.752976:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205222.752977:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205222.752979:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205222.752981:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526528:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205222.752984:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205222.752987:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205222.752988:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.752990:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205222.752993:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205222.752995:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205222.752998:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205222.753000:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205222.753003:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205222.753005:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543526528, portal 4 00000100:00000001:3.0:1788205222.753007:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205222.753008:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205222.753013:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753019:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205222.753021:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205222.753023:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205222.753026:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543526528, offset 0 00000400:00000200:3.0:1788205222.753030:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205222.753033:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.753037:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205222.753039:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205222.753041:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205222.753043:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205222.753045:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205222.753049:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471462:471462:256:4294967295] 192.168.201.118@tcp LPNI seq info [471321:471321:8:4294967295] 00000400:00000200:3.0:1788205222.753053:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.753059:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205222.753063:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.753066:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205222.753070:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205222.753072:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205222.753076:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205222.753078:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205222.753082:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.753084:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205222.753086:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205222.753109:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205222.753112:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205222.753113:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205222.753116:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205222.753119:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753126:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.753129:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753134:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.753136:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205222.753138:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205222.753141:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205222.753144:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205222.753146:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321581601 00000100:00000001:3.0:1788205222.753148:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321582857 00000100:00000001:3.0:1788205222.753150:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321581601 00000100:00000001:3.0:1788205222.753152:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321582857 00000100:00000001:3.0:1788205222.753154:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205222.753157:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.753158:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205222.753162:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205222/real 1788205222] req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205222.753170:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205222.753171:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205222.753173:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205222.753175:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205222.753176:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205222.753178:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205222.753181:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205222.753189:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753195:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205222.753196:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753198:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.753199:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205222.753203:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753210:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205222.753213:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753219:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205222.753221:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753223:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.753224:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753226:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205222.753227:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205222.753230:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205222.753233:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205222.753235:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205222.753239:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205222.753246:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526528:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205222.753249:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205222.753253:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753259:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205222.753260:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.753262:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205222.753264:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205222.753266:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205222.753269:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526656:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205222.753272:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205222.753273:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205222.753275:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205222.753276:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205222.753279:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205222.753282:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205222.753284:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205222.753287:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205222.753289:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205222.753292:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543526656, portal 4 00000100:00000001:3.0:1788205222.753294:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205222.753295:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205222.753300:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753306:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205222.753309:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205222.753311:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205222.753313:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543526656, offset 0 00000400:00000200:3.0:1788205222.753317:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205222.753320:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.753324:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205222.753327:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205222.753329:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205222.753331:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205222.753333:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205222.753336:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471463:471463:256:4294967295] 192.168.201.118@tcp LPNI seq info [471322:471322:8:4294967295] 00000400:00000200:3.0:1788205222.753341:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205222.753347:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205222.753351:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.753354:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205222.753358:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205222.753360:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205222.753364:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205222.753367:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205222.753370:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205222.753372:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205222.753375:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205222.753378:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205222.753381:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205222.753382:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205222.753384:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205222.753387:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753395:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.753397:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205247 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753403:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.753405:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205222.753407:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205222.753410:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205222.753413:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205222.753415:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321581624 00000100:00000001:3.0:1788205222.753417:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321582857 00000100:00000001:3.0:1788205222.753419:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321581624 00000100:00000001:3.0:1788205222.753421:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321582857 00000100:00000001:3.0:1788205222.753423:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205222.753425:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.753427:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205222.753430:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205222/real 1788205222] req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205222.753439:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205222.753440:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205222.753442:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205222.753443:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205222.753445:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205222.753446:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205222.753449:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205222.753458:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753464:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205222.753465:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753467:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.753468:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205222.753472:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753478:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205222.753481:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753488:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205222.753490:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753492:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205222.753493:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753495:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205222.753497:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205222.753499:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205222.753502:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205222.753504:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205222.753507:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205222.753514:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526656:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205222.753517:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753519:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.753522:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543526144/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753529:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205222.753533:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205222.753536:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205222.753539:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205222.753541:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205222.753543:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205222.753545:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205222.753549:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:3.0:1788205222.753553:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205222.753555:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205222.753557:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205222.753565:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205222.753566:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753569:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.753572:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543526272/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753579:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205222.753581:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205222.753582:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205222.753585:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205222.753586:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205222.753587:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205222.753589:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205222.753590:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:3.0:1788205222.753593:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205222.753594:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205222.753595:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205222.753598:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205222.753599:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753600:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.753603:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543526400/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753610:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205222.753611:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205222.753612:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205222.753614:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205222.753615:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205222.753616:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205222.753617:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205222.753618:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:3.0:1788205222.753620:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205222.753621:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205222.753622:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205222.753641:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205222.753643:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753644:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.753647:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543526528/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753659:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205222.753660:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205222.753662:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205222.753663:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205222.753665:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205222.753666:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205222.753668:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205222.753669:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:3.0:1788205222.753671:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205222.753672:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205222.753673:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205222.753675:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205222.753676:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753678:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205222.753680:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543526656/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205247 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205222.753686:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205222.753687:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205222.753688:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205222.753691:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205222.753692:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205222.753693:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205222.753695:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205222.753696:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:3.0:1788205222.753698:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205222.753699:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205222.753700:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205222.753714:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205222.753715:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753718:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205222.753739:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205222.753741:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205222.753744:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205222.753746:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205223.580251:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205223.580262:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205223.580264:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205223.580271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205223.580275:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205223.580280:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205223.580283:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205223.580287:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205223.580290:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205223.582516:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205223.582523:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205223.582528:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205223.582529:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205223.582532:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205223.582534:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205223.582537:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205223.582551:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205223.582629:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205223.582634:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205223.582639:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205223.582984:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205223.582989:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205223.582991:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205223.582995:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205223.582999:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205223.583002:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205223.583005:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205223.583007:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205223.583014:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205224.092256:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321582857, at 4321582912 00000100:00000010:3.0:1788205224.092276:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf60. 00000100:00000040:2.0:1788205224.092371:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321582857, at 4321582912 00000100:00000010:2.0:1788205224.092377:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc300. 00000100:00000001:2.0:1788205224.604198:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205224.604206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205224.604208:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205224.604213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205224.604213:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205224.604222:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205224.604226:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205224.604229:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205224.604230:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205224.604234:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205224.604235:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205224.604241:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205224.604242:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205224.604245:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205224.604246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604251:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205224.604252:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604254:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205224.604262:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205224.604263:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205224.604264:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604265:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205224.604266:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604268:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604271:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205224.604277:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205224.604279:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205224.604280:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604282:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205224.604283:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604284:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205224.604286:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205224.604402:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205224.604406:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205224.604409:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205224.604464:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205224.604468:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205224.604470:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205224.604473:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205224.604475:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205224.604477:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205224.604480:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205224.604481:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205224.604484:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205225.628241:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205225.628252:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205225.628254:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205225.628261:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205225.628265:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205225.628269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205225.628273:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205225.628276:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205225.628279:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205225.628367:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205225.628372:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205225.628373:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205225.628377:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205225.628379:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205225.628381:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205225.628384:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205225.628387:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205225.628389:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205225.628491:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205225.628494:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205225.628499:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205225.628501:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205225.628504:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205225.628505:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205225.628508:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205225.628509:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205225.628645:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205225.628649:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205225.628653:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205226.652284:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205226.652295:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205226.652297:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205226.652304:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205226.652309:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205226.652313:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205226.652317:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205226.652320:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205226.652323:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205226.652445:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205226.652450:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205226.652454:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205226.652456:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205226.652458:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205226.652460:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205226.652463:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205226.652464:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205226.652527:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205226.652531:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205226.652533:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205226.652535:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205226.652538:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205226.652540:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205226.652542:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205226.652544:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205226.652546:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205226.652736:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205226.652741:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205226.652744:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205227.676247:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205227.676259:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205227.676262:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205227.676271:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205227.676276:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205227.676280:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.676280:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205227.676284:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205227.676288:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.676288:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205227.676292:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205227.676293:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205227.676298:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205227.676303:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205227.676306:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.676309:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205227.676311:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.676314:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205227.676533:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205227.676539:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205227.676545:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205227.676546:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205227.676550:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205227.676551:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205227.676555:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205227.676556:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205227.676624:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205227.676629:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205227.676632:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205227.868414:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205227.868429:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205227.868433:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205227.868437:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205227.868442:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205227.868452:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26914 00000400:00000200:1.0:1788205227.868465:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205227.868474:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205227.868478:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205227.868482:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868485:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.868488:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868490:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205227.868495:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205227.868497:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205227.868501:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205227.868505:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205227.868508:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205227.868511:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205227.868514:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868517:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.868519:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868520:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205227.868524:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205227.868529:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868531:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205227.868535:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205227.868540:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205227.868543:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.868546:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205227.868549:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205227.868560:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788205227.868564:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205227.868569:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205227.868570:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.868575:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205227.868579:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205227.868584:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205227.868593:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8800. 00000100:00000001:1.0:1788205227.868601:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205227.868610:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205227.868621:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205227.868626:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205227.868631:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205227.868665:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205227.868668:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205227.868672:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205227.868674:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205227.868675:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205227.868678:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205227.868681:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26914 00000400:00000200:1.0:1788205227.868685:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205227.868688:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205227.868691:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205227.868694:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868696:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.868698:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868700:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205227.868702:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205227.868704:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205227.868705:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205227.868707:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205227.868710:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205227.868712:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205227.868715:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868717:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.868719:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868721:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205227.868724:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205227.868726:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868729:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205227.868730:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205227.868737:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205227.868739:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.868741:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205227.868745:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205227.868749:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.868751:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205227.868754:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205227.868757:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205227.868760:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205227.868761:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.868763:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205227.868764:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205227.868766:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205227.868769:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8400. 00000100:00000001:1.0:1788205227.868773:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205227.868778:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205227.868786:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205227.868788:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205227.868791:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205227.868799:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205227.868802:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205227.868805:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205227.868807:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205227.868808:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205227.868811:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205227.868814:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26914 00000400:00000200:1.0:1788205227.868818:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205227.868821:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205227.868824:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205227.868826:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868828:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.868830:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868832:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205227.868835:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205227.868836:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205227.868838:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205227.868840:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205227.868853:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205227.868855:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205227.868859:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868861:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.868863:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868865:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205227.868867:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205227.868870:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868872:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205227.868874:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205227.868876:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205227.868878:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.868880:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205227.868882:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205227.868884:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.868885:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205227.868891:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205227.868893:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205227.868895:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205227.868896:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.868898:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205227.868899:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205227.868901:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205227.868905:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb000. 00000100:00000001:1.0:1788205227.868909:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205227.868913:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205227.868919:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205227.868921:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205227.868924:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205227.868932:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205227.868935:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205227.868937:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205227.868939:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205227.868941:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205227.868944:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205227.868947:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26914 00000400:00000200:1.0:1788205227.868951:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205227.868954:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205227.868956:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205227.868959:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868961:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.868963:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868965:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205227.868967:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205227.868969:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205227.868970:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205227.868972:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205227.868974:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205227.868976:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205227.868979:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868981:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.868983:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868985:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205227.868987:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205227.868990:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205227.868992:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205227.868994:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205227.868997:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205227.868999:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.869001:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205227.869003:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205227.869005:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.869007:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205227.869011:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205227.869014:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205227.869016:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205227.869017:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.869018:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205227.869020:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205227.869021:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205227.869025:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9c00. 00000100:00000001:1.0:1788205227.869028:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205227.869031:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205227.869038:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205227.869040:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205227.869044:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205227.869052:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205227.869054:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205227.869057:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205227.869059:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205227.869060:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205227.869064:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205227.869066:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26914 00000400:00000200:1.0:1788205227.869070:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205227.869072:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205227.869075:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205227.869077:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.869078:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.869080:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.869083:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205227.869084:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205227.869085:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205227.869088:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205227.869112:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205227.869114:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205227.869116:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205227.869119:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.869121:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205227.869123:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205227.869125:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205227.869127:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205227.869129:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205227.869131:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205227.869133:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205227.869136:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205227.869138:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.869139:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205227.869141:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205227.869143:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.869145:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205227.869149:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000020:00000040:1.0:1788205227.869150:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205227.869152:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205227.869154:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205227.869156:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205227.869157:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205227.869159:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205227.869162:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9800. 00000100:00000001:1.0:1788205227.869165:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205227.869169:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205227.869175:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205227.869177:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205227.869181:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205227.869189:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205227.869194:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26924) 00000100:00000001:3.0:1788205227.869968:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205227.869975:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205227.869978:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205227.869987:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.869996:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205227.869999:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.870003:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205227.870007:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205227.870009:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205227.870013:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543526784:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205227.870018:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205227.870023:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205227.870026:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.870029:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205227.870035:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205227.870037:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205227.870043:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205227.870047:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205227.870056:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205227.870064:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543526784, portal 25 00000100:00000001:3.0:1788205227.870067:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205227.870068:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000040:3.0:1788205227.870074:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870082:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205227.870084:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205227.870087:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205227.870134:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543526784, offset 0 00000400:00000200:3.0:1788205227.870161:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205227.870171:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.870179:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205227.870185:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205227.870189:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205227.870194:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205227.870198:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205227.870203:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471464:471464:256:4294967295] 192.168.201.118@tcp LPNI seq info [471323:471323:8:4294967295] 00000400:00000200:3.0:1788205227.870208:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.870272:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205227.870280:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.870287:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205227.870296:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205227.870301:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205227.870307:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205227.870310:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205227.870315:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.870318:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205227.870325:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205227.870330:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205227.870336:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205227.870339:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205227.870344:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205227.870348:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870357:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.870361:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870367:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.870369:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205227.870376:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205227.870381:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205227.870384:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321587975 00000100:00000010:3.0:1788205227.870390:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcf60. 00000100:00000001:3.0:1788205227.870399:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205227.870405:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321587975 00000100:00000010:3.0:1788205227.870409:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcd20. 00000100:00000001:3.0:1788205227.870412:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205227.870415:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205227.870418:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.870420:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205227.870425:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205227/real 1788205227] req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205227.870436:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205227.870438:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205227.870440:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205227.870443:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205227.870445:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205227.870448:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205227.870451:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205227.870460:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870467:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205227.870469:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.870472:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.870474:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205227.870478:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870484:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205227.870488:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870495:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205227.870498:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.870500:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.870502:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.870504:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205227.870506:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205227.870510:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205227.870515:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205227.870518:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205227.870522:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205227.870531:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543526784:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205227.870536:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205227.870539:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870546:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205227.870548:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.870550:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205227.870552:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205227.870555:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205227.870557:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526912:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205227.870561:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205227.870564:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205227.870566:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.870568:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205227.870571:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205227.870573:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205227.870577:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205227.870579:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205227.870583:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205227.870586:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543526912, portal 10 00000100:00000001:3.0:1788205227.870588:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205227.870590:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788205227.870596:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870603:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205227.870605:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205227.870607:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205227.870609:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543526912, offset 0 00000400:00000200:3.0:1788205227.870613:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205227.870617:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.870621:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205227.870624:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205227.870626:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205227.870628:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205227.870631:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205227.870634:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471465:471465:256:4294967295] 192.168.201.118@tcp LPNI seq info [471324:471324:8:4294967295] 00000400:00000200:3.0:1788205227.870638:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.870644:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205227.870648:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.870652:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205227.870656:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205227.870659:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205227.870663:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205227.870665:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205227.870669:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.870671:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205227.870674:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205227.870677:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205227.870680:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205227.870682:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205227.870685:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205227.870689:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870702:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.870705:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870711:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.870712:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205227.870715:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205227.870720:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205227.870723:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205227.870725:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321586708 00000100:00000001:3.0:1788205227.870728:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321587975 00000100:00000001:3.0:1788205227.870730:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321586708 00000100:00000001:3.0:1788205227.870732:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321587975 00000100:00000001:3.0:1788205227.870734:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205227.870736:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.870737:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205227.870741:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205227/real 1788205227] req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205227.870749:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205227.870751:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205227.870752:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205227.870754:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205227.870756:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205227.870758:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205227.870761:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205227.870769:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870776:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205227.870777:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.870779:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.870781:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205227.870784:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870790:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205227.870794:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870801:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205227.870803:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.870805:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.870806:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.870808:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205227.870810:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205227.870813:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205227.870816:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205227.870818:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205227.870822:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205227.870830:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543526912:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205227.870834:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205227.870838:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870855:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205227.870857:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.870859:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205227.870861:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205227.870863:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205227.870867:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527040:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205227.870870:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205227.870872:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205227.870874:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.870876:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205227.870880:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205227.870882:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205227.870885:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205227.870887:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205227.870891:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205227.870893:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543527040, portal 10 00000100:00000001:3.0:1788205227.870895:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205227.870897:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205227.870903:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870909:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205227.870911:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205227.870913:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205227.870915:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543527040, offset 0 00000400:00000200:3.0:1788205227.870919:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205227.870923:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.870926:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205227.870929:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205227.870931:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205227.870933:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205227.870935:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205227.870938:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471466:471466:256:4294967295] 192.168.201.118@tcp LPNI seq info [471325:471325:8:4294967295] 00000400:00000200:3.0:1788205227.870942:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.870949:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205227.870953:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.870956:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205227.870960:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205227.870962:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205227.870966:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205227.870969:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205227.870972:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.870974:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205227.870977:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205227.870980:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205227.870983:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205227.870985:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205227.870987:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205227.870991:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.870998:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.871001:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.871007:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.871009:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205227.871012:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205227.871015:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205227.871018:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205227.871020:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321586709 00000100:00000001:3.0:1788205227.871022:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321587975 00000100:00000001:3.0:1788205227.871024:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321586709 00000100:00000001:3.0:1788205227.871026:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321587975 00000100:00000001:3.0:1788205227.871028:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205227.871030:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.871032:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205227.871036:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205227/real 1788205227] req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205227.871044:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205227.871046:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205227.871048:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205227.871050:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205227.871051:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205227.871054:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205227.871057:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205227.871065:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.871071:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205227.871072:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.871074:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.871076:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205227.871079:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.871085:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205227.871110:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.871118:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205227.871121:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.871123:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.871124:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.871126:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205227.871129:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205227.871131:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205227.871135:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205227.871137:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205227.871141:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205227.871154:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527040:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205227.871158:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205227.871162:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.871169:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205227.871170:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.871173:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205227.871174:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205227.871176:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205227.871179:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527168:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205227.871183:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205227.871185:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205227.871186:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.871188:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205227.871191:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205227.871193:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205227.871197:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205227.871199:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205227.871203:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205227.871206:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543527168, portal 4 00000100:00000001:3.0:1788205227.871208:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205227.871209:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205227.872307:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872323:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205227.872326:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205227.872332:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205227.872337:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543527168, offset 0 00000400:00000200:3.0:1788205227.872344:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205227.872351:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.872358:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205227.872362:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205227.872365:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205227.872369:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205227.872371:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205227.872374:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471467:471467:256:4294967295] 192.168.201.118@tcp LPNI seq info [471326:471326:8:4294967295] 00000400:00000200:3.0:1788205227.872378:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.872385:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205227.872391:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.872396:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205227.872402:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205227.872406:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205227.872411:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205227.872414:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205227.872418:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.872420:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205227.872425:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205227.872429:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205227.872433:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205227.872436:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205227.872439:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205227.872445:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872455:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.872458:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872464:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.872466:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205227.872470:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205227.872474:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205227.872477:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205227.872480:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321586720 00000100:00000001:3.0:1788205227.872483:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321587975 00000100:00000001:3.0:1788205227.872485:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321586720 00000100:00000001:3.0:1788205227.872487:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321587975 00000100:00000001:3.0:1788205227.872489:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205227.872492:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.872494:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205227.872499:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205227/real 1788205227] req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205227.872508:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205227.872509:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205227.872511:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205227.872514:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205227.872516:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205227.872518:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205227.872522:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205227.872530:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872536:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205227.872538:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.872540:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.872542:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205227.872546:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872553:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205227.872557:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872564:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205227.872568:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.872570:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.872571:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.872573:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205227.872576:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205227.872579:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205227.872583:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205227.872586:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205227.872590:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205227.872598:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527168:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205227.872603:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205227.872607:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872613:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205227.872615:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.872619:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205227.872622:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205227.872625:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205227.872628:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527296:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205227.872632:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205227.872635:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205227.872638:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205227.872639:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205227.872643:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205227.872645:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205227.872649:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205227.872653:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205227.872657:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205227.872661:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543527296, portal 4 00000100:00000001:3.0:1788205227.872663:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205227.872664:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205227.872670:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872677:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205227.872679:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205227.872682:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205227.872684:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543527296, offset 0 00000400:00000200:3.0:1788205227.872687:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205227.872691:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.872695:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205227.872698:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205227.872700:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205227.872702:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205227.872704:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205227.872707:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471468:471468:256:4294967295] 192.168.201.118@tcp LPNI seq info [471327:471327:8:4294967295] 00000400:00000200:3.0:1788205227.872711:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205227.872717:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205227.872721:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.872724:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205227.872728:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205227.872731:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:3.1:1788205227.872735:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205227.872738:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:3.0:1788205227.872741:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205227.872743:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:3.0:1788205227.872746:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205227.872749:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205227.872752:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:3.0:1788205227.872754:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:3.0:1788205227.872756:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205227.872759:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872772:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.872775:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205252 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872781:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.872783:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205227.872786:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205227.872788:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205227.872792:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205227.872794:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321586743 00000100:00000001:3.0:1788205227.872796:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321587975 00000100:00000001:3.0:1788205227.872798:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321586743 00000100:00000001:3.0:1788205227.872800:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321587975 00000100:00000001:3.0:1788205227.872802:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205227.872804:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.872806:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205227.872809:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205227/real 1788205227] req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205227.872817:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205227.872818:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205227.872820:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205227.872822:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205227.872824:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205227.872826:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205227.872829:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205227.872837:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872858:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205227.872860:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.872861:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.872863:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205227.872866:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872873:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205227.872876:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872883:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205227.872885:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.872887:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205227.872888:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.872890:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205227.872892:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205227.872895:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205227.872899:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205227.872901:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205227.872905:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205227.872913:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527296:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205227.872918:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.872921:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.872925:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543526784/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872932:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205227.872935:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205227.872937:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205227.872941:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205227.872945:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205227.872947:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205227.872950:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205227.872953:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:3.0:1788205227.872957:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205227.872960:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205227.872962:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205227.872970:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205227.872971:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.872973:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.872978:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543526912/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.872984:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205227.872985:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205227.872987:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205227.872989:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205227.872991:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205227.872992:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205227.872994:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205227.872995:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:3.0:1788205227.872997:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205227.872998:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205227.873000:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788205227.873005:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205227.873007:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.873008:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.873012:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543527040/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.873019:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205227.873020:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205227.873022:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:3.0:1788205227.873024:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205227.873025:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205227.873026:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205227.873028:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205227.873030:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:3.0:1788205227.873032:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205227.873033:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205227.873035:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205227.873037:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205227.873038:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.873040:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.873043:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543527168/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.873050:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205227.873051:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205227.873053:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:3.0:1788205227.873054:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205227.873055:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205227.873056:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205227.873057:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205227.873059:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:3.0:1788205227.873061:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205227.873062:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205227.873063:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205227.873070:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205227.873071:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.873072:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205227.873076:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543527296/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205252 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205227.873082:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205227.873083:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205227.873085:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:3.0:1788205227.873087:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205227.873088:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205227.873110:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205227.873112:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205227.873113:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:3.0:1788205227.873116:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205227.873117:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205227.873118:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205227.873121:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205227.873122:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.873126:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205227.873135:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205227.873137:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205227.873140:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205227.873143:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205228.700262:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205228.700275:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205228.700277:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205228.700284:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205228.700288:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205228.700294:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205228.700297:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205228.700301:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205228.700304:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205228.700423:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205228.700427:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205228.700433:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205228.700435:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205228.700438:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205228.700439:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205228.700442:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205228.700443:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205228.700686:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205228.700692:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205228.700697:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205228.700719:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205228.700725:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205228.700727:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205228.700730:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205228.700732:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205228.700735:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205228.700737:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205228.700739:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205228.700741:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205229.212230:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321587975, at 4321588032 00000100:00000040:3.0:1788205229.212230:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321587975, at 4321588032 00000100:00000010:2.0:1788205229.212250:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcf60. 00000100:00000010:3.0:1788205229.212264:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcd20. 00010000:00000001:0.0:1788205229.724231:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205229.724240:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205229.724243:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205229.724245:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205229.724249:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205229.724251:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205229.724252:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205229.724254:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205229.724257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205229.724259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205229.724262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205229.724263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205229.724265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205229.724266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205229.724297:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205229.724308:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205229.724310:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205229.724317:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205229.724322:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788205229.724323:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000100:00000001:1.0:1788205229.724327:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788205229.724329:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205229.724333:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00010000:2.0:1788205229.724335:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205229.724338:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205229.724345:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205229.724348:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205229.724355:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205229.724358:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205229.724362:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205229.724366:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205229.724370:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205229.724373:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205229.724376:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205229.724378:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205230.748262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205230.748270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205230.748277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205230.748278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205230.748282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205230.748284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205230.748284:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205230.748288:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205230.748290:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205230.748291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205230.748294:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205230.748299:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205230.748303:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205230.748306:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205230.748309:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205230.748312:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205230.748315:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205230.748744:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205230.748748:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205230.748752:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205230.748891:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205230.748895:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205230.748896:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205230.748898:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205230.748901:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205230.748903:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205230.748905:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205230.748907:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205230.748909:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205231.772585:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205231.772593:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205231.772599:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205231.772600:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205231.772603:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205231.772604:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205231.772607:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205231.772608:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205231.772869:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205231.772873:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205231.772876:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205231.772927:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205231.772932:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205231.772934:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.772936:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00100000:1.0:1788205231.772939:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205231.772941:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205231.772945:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205231.772945:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205231.772949:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205231.772950:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205231.772954:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205231.772955:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205231.772958:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205231.772959:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205231.772964:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205231.772965:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205231.772968:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205231.772972:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.796186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205232.796194:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205232.796195:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205232.796197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205232.796200:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205232.796205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205232.796206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205232.796209:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.796211:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205232.796215:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205232.796216:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205232.796218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.796220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205232.796223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205232.796224:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205232.796225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.796230:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205232.796338:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205232.796342:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205232.796345:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205232.796397:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205232.796401:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205232.796402:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205232.796405:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205232.796408:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205232.796410:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205232.796412:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205232.796414:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205232.796417:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205232.988738:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205232.988747:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205232.988750:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205232.988796:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205232.988801:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205232.988808:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26919 00000400:00000200:3.0:1788205232.988819:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205232.988825:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205232.988829:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205232.988831:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.988834:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.988837:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.988840:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205232.988844:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205232.988846:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205232.988848:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205232.988860:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205232.988862:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205232.988865:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205232.988868:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.988870:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.988872:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.988874:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205232.988878:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205232.988881:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205232.988884:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205232.988886:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205232.988889:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205232.988892:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.988894:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205232.988896:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205232.988907:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788205232.988910:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205232.988914:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205232.988915:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.988918:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205232.988921:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205232.988925:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205232.988931:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788205232.988937:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205232.988945:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205232.988956:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205232.988960:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205232.988965:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989050:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.989054:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205232.989057:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205232.989063:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989071:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205232.989073:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.989076:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205232.989078:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205232.989081:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205232.989084:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543527424:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205232.989088:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205232.989108:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205232.989110:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.989113:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205232.989118:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.989120:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205232.989125:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205232.989129:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205232.989139:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205232.989147:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543527424, portal 25 00000100:00000001:1.0:1788205232.989149:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205232.989151:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205232.989156:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989161:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205232.989164:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205232.989166:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205232.989170:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543527424, offset 0 00000400:00000200:1.0:1788205232.989176:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205232.989184:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.989190:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205232.989195:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205232.989199:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205232.989202:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205232.989205:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205232.989209:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471469:471469:256:4294967295] 192.168.201.118@tcp LPNI seq info [471328:471328:8:4294967295] 00000400:00000200:1.0:1788205232.989213:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.989223:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205232.989228:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.989233:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205232.989239:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205232.989243:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205232.989247:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205232.989251:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205232.989256:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.989258:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205232.989265:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205232.989270:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205232.989276:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205232.989278:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205232.989283:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205232.989287:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205232.989336:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205232.989340:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205232.989343:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205232.989344:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205232.989346:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205232.989349:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205232.989353:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26919 00000400:00000200:3.0:1788205232.989357:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205232.989360:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205232.989362:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205232.989365:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.989367:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.989369:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.989371:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205232.989373:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205232.989375:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205232.989376:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205232.989378:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205232.989381:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205232.989383:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205232.989386:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.989387:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.989388:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.989390:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205232.989392:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205232.989395:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205232.989397:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205232.989399:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205232.989404:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205232.989406:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.989407:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205232.989411:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205232.989414:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.989416:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205232.989419:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205232.989421:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205232.989424:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205232.989425:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.989427:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205232.989428:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205232.989430:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205232.989434:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 00000100:00000001:3.0:1788205232.989437:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205232.989442:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205232.989449:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205232.989451:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205232.989455:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989521:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.989536:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989544:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.989545:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205232.989553:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205232.989557:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205232.989560:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321593094 00000100:00000010:1.0:1788205232.989567:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647180. 00000100:00000001:1.0:1788205232.989574:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205232.989579:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321593094 00000100:00000010:1.0:1788205232.989582:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647000. 00000100:00000001:1.0:1788205232.989585:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205232.989588:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205232.989591:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.989594:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205232.989597:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205232/real 1788205232] req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205232.989607:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205232.989608:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205232.989610:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205232.989612:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205232.989614:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205232.989616:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205232.989619:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205232.989629:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989635:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205232.989637:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.989639:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.989640:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205232.989645:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989651:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205232.989655:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989662:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205232.989665:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.989666:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.989667:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.989682:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205232.989685:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205232.989688:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205232.989692:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205232.989694:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205232.989699:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205232.989707:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543527424:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205232.989711:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.989713:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.989716:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543527424/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989724:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205232.989726:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205232.989728:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.989731:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205232.989733:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205232.989735:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205232.989738:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205232.989741:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205232.989744:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205232.989746:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205232.989747:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205232.989751:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205232.989760:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.989762:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.989769:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205232.989770:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.989773:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.989774:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205232.989776:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205232.989779:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989785:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205232.989787:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.989789:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205232.989791:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205232.989793:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205232.989795:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527552:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205232.989798:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205232.989800:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205232.989802:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.989803:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205232.989806:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.989808:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205232.989812:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205232.989814:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205232.989818:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205232.989820:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543527552, portal 10 00000100:00000001:1.0:1788205232.989822:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205232.989823:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205232.989829:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989835:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205232.989837:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205232.989839:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205232.989841:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543527552, offset 0 00000400:00000200:1.0:1788205232.989845:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205232.989849:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.989862:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205232.989865:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205232.989867:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205232.989870:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205232.989872:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205232.989875:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471470:471470:256:4294967295] 192.168.201.118@tcp LPNI seq info [471329:471329:8:4294967295] 00000400:00000200:1.0:1788205232.989880:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.989886:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205232.989891:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.989894:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205232.989897:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205232.989900:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205232.989904:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205232.989907:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205232.989910:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.989912:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205232.989915:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205232.989918:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205232.989921:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205232.989923:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205232.989925:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205232.989928:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989935:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.989939:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.989945:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.989947:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205232.989949:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205232.989953:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205232.989955:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205232.989957:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321591827 00000100:00000001:1.0:1788205232.989960:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321593094 00000100:00000001:1.0:1788205232.989962:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321591827 00000100:00000001:1.0:1788205232.989965:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321593094 00000100:00000001:1.0:1788205232.989967:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205232.989968:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.989970:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205232.989974:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205232/real 1788205232] req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205232.989982:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205232.989983:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205232.989984:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205232.989986:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205232.989988:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205232.989989:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205232.989992:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205232.990000:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.990006:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205232.990008:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.990009:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.990010:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205232.990013:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.990019:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205232.990023:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.990029:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205232.990032:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.990033:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.990034:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.990037:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205232.990039:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205232.990041:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205232.990044:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205232.990046:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205232.990050:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205232.990058:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527552:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205232.990062:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.990063:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.990066:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543527552/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.990073:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205232.990075:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205232.990076:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.990078:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205232.990080:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205232.990081:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205232.990083:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205232.990084:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205232.990086:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205232.990088:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205232.990089:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205232.990119:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205232.990121:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.990122:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.990126:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205232.990127:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.990129:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.990130:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.990740:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.990743:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.990789:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205232.990792:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205232.990795:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205232.990796:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205232.990798:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205232.990801:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205232.990804:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26919 00000400:00000200:3.0:1788205232.990808:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205232.990812:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205232.990815:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205232.990817:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.990820:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.990822:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.990824:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205232.990827:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205232.990828:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205232.990830:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205232.990832:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205232.990834:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205232.990836:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205232.990838:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.990840:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.990842:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.990844:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205232.990847:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205232.990849:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205232.990910:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205232.990912:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205232.990916:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205232.990918:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.990921:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205232.990923:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205232.990926:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.990927:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205232.990932:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788205232.990934:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205232.990937:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205232.990939:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.990940:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205232.990942:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205232.990943:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205232.990947:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788205232.990951:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205232.990956:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205232.990964:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205232.990966:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205232.990969:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991010:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.991013:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205232.991015:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205232.991019:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991026:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205232.991027:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.991029:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205232.991031:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205232.991033:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205232.991037:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527680:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205232.991040:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205232.991042:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205232.991043:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.991045:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205232.991049:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.991051:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205232.991054:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205232.991057:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205232.991061:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205232.991065:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543527680, portal 10 00000100:00000001:1.0:1788205232.991067:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205232.991068:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205232.991074:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991081:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205232.991083:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205232.991086:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205232.991089:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543527680, offset 0 00000400:00000200:1.0:1788205232.991108:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205232.991112:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.991116:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205232.991119:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205232.991121:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205232.991124:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205232.991126:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205232.991129:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471471:471471:256:4294967295] 192.168.201.118@tcp LPNI seq info [471330:471330:8:4294967295] 00000400:00000200:1.0:1788205232.991133:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.991139:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205232.991143:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.991146:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205232.991150:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205232.991153:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205232.991158:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205232.991161:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205232.991164:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.991166:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205232.991169:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205232.991172:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205232.991175:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205232.991177:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205232.991180:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205232.991183:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205232.991216:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205232.991220:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205232.991224:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205232.991225:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205232.991227:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205232.991229:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205232.991232:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26919 00000400:00000200:3.0:1788205232.991236:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205232.991239:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205232.991242:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205232.991244:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.991246:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.991248:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.991250:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205232.991252:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205232.991254:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205232.991255:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205232.991258:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205232.991260:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205232.991261:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205232.991264:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.991266:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.991268:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.991270:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205232.991273:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205232.991275:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205232.991278:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205232.991279:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205232.991282:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205232.991283:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.991285:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205232.991288:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205232.991291:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.991293:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205232.991296:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205232.991298:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205232.991301:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205232.991302:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.991304:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205232.991305:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205232.991307:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205232.991310:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000100:00000001:3.0:1788205232.991314:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205232.991318:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205232.991325:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205232.991327:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205232.991331:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991358:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.991362:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991368:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.991369:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205232.991373:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205232.991377:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205232.991380:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205232.991382:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321591830 00000100:00000001:1.0:1788205232.991385:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321593094 00000100:00000001:1.0:1788205232.991386:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321591830 00000100:00000001:1.0:1788205232.991389:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321593094 00000100:00000001:1.0:1788205232.991391:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205232.991393:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.991395:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205232.991398:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205232/real 1788205232] req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205232.991405:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205232.991406:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205232.991408:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205232.991410:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205232.991412:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205232.991414:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205232.991416:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205232.991424:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991431:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205232.991432:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.991434:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.991435:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205232.991439:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991445:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205232.991449:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991456:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205232.991458:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.991459:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.991460:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.991462:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205232.991464:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205232.991467:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205232.991470:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205232.991473:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205232.991476:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205232.991483:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527680:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205232.991487:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.991489:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.991492:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543527680/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991497:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205232.991499:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205232.991501:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.991503:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205232.991505:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205232.991506:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205232.991508:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205232.991510:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205232.991512:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205232.991513:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205232.991514:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205232.991516:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205232.991517:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.991519:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.991523:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205232.991727:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.991730:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.991732:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205232.991733:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205232.991738:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991746:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205232.991747:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.991749:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205232.991751:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205232.991761:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205232.991765:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527808:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205232.991768:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205232.991770:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205232.991772:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.991773:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205232.991777:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.991778:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205232.991782:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205232.991784:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205232.991788:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205232.991792:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543527808, portal 4 00000100:00000001:1.0:1788205232.991794:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205232.991795:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205232.991800:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991807:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205232.991809:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205232.991811:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205232.991813:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543527808, offset 0 00000400:00000200:1.0:1788205232.991817:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205232.991820:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.991824:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205232.991827:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205232.991829:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205232.991831:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205232.991834:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205232.991837:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471472:471472:256:4294967295] 192.168.201.118@tcp LPNI seq info [471331:471331:8:4294967295] 00000400:00000200:1.0:1788205232.991842:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.991848:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205232.991852:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.991910:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205232.991915:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205232.991918:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205232.991923:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205232.991925:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205232.991928:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.991930:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205232.991933:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205232.991936:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205232.991940:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205232.991942:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205232.991944:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205232.991948:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991955:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.991958:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.991965:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.991966:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205232.991969:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205232.991972:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205232.991975:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205232.991977:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321591839 00000100:00000001:1.0:1788205232.991979:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321593094 00000100:00000001:1.0:1788205232.991981:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321591839 00000100:00000001:1.0:1788205232.991983:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321593094 00000100:00000001:1.0:1788205232.991985:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205232.991987:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.991989:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205232.991992:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205232/real 1788205232] req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205232.992000:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205232.992002:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205232.992003:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205232.992005:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205232.992007:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205232.992009:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205232.992011:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205232.992018:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992024:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205232.992026:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992027:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.992029:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205232.992032:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992039:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205232.992043:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992049:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205232.992051:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992052:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.992054:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992055:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205232.992057:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205232.992060:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205232.992063:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205232.992065:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205232.992069:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205232.992075:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527808:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205232.992078:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992080:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.992083:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543527808/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992088:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205232.992105:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205232.992107:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.992109:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205232.992111:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205232.992112:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205232.992114:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205232.992115:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205232.992117:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205232.992118:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205232.992120:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205232.992122:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205232.992123:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992124:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992128:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205232.992129:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.992131:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.992132:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.992190:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.992192:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.992220:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205232.992223:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205232.992227:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205232.992228:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205232.992230:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205232.992232:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205232.992235:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26919 00000400:00000200:3.0:1788205232.992239:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205232.992243:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205232.992245:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205232.992248:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.992250:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.992252:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.992255:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205232.992257:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205232.992259:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205232.992260:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205232.992262:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205232.992264:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205232.992266:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205232.992268:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.992270:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205232.992272:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205232.992274:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205232.992276:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205232.992279:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205232.992281:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205232.992283:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205232.992286:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205232.992287:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.992289:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205232.992291:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205232.992294:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.992295:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205232.992300:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205232.992302:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205232.992304:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205232.992305:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205232.992307:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205232.992309:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205232.992310:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205232.992314:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000001:3.0:1788205232.992318:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205232.992322:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205232.992335:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205232.992337:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205232.992341:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992375:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.992377:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205232.992379:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205232.992383:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992390:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205232.992391:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.992393:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205232.992395:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205232.992397:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205232.992400:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527936:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205232.992403:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205232.992405:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205232.992406:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205232.992408:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205232.992411:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.992413:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205232.992416:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205232.992418:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205232.992421:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205232.992424:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543527936, portal 4 00000100:00000001:1.0:1788205232.992425:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205232.992427:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205232.992432:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992438:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205232.992440:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205232.992442:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205232.992444:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543527936, offset 0 00000400:00000200:1.0:1788205232.992448:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205232.992451:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.992455:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205232.992457:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205232.992460:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205232.992462:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205232.992465:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205232.992468:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471473:471473:256:4294967295] 192.168.201.118@tcp LPNI seq info [471332:471332:8:4294967295] 00000400:00000200:1.0:1788205232.992472:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205232.992478:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205232.992482:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.992485:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205232.992489:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205232.992492:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000200:1.1:1788205232.992495:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205232.992498:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 2 secs later) 00000800:00000100:1.0:1788205232.992501:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205232.992503:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205232.992506:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205232.992509:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205232.992512:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205232.992513:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205232.992515:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205232.992518:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205232.992541:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205232.992547:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26929) 00000100:00000001:1.0:1788205232.992579:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.992583:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205257 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992590:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.992591:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205232.992595:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205232.992598:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205232.992602:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205232.992604:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321591863 00000100:00000001:1.0:1788205232.992606:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321593094 00000100:00000001:1.0:1788205232.992608:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321591863 00000100:00000001:1.0:1788205232.992610:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321593094 00000100:00000001:1.0:1788205232.992612:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205232.992614:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.992616:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205232.992619:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205232/real 1788205232] req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205232.992628:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205232.992629:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205232.992630:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205232.992632:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205232.992634:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205232.992636:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205232.992639:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205232.992646:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992652:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205232.992653:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992655:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.992656:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205232.992659:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992665:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205232.992669:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992722:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205232.992724:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992726:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205232.992727:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992729:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205232.992730:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205232.992733:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205232.992737:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205232.992739:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205232.992743:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205232.992750:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543527936:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205232.992779:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992780:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205232.992784:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543527936/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205257 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205232.992790:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205232.992791:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205232.992793:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:1.0:1788205232.992795:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205232.992797:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205232.992798:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205232.992800:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205232.992802:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205232.992804:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205232.992805:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205232.992806:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205232.992808:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205232.992810:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992811:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205232.992814:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205232.992816:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205232.992818:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205232.992819:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205233.820187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205233.820196:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205233.820198:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205233.820204:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205233.820208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205233.820211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205233.820214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205233.820216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205233.820220:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205233.820317:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205233.820321:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205233.820326:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205233.820327:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205233.820339:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205233.820341:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205233.820343:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205233.820345:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205233.820433:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205233.820436:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205233.820438:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205233.820441:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205233.820443:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205233.820445:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205233.820447:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205233.820449:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205233.820451:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205233.820493:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205233.820497:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205233.820500:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205234.332277:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321593094, at 4321593152 00000100:00000040:2.0:1788205234.332278:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321593094, at 4321593152 00000100:00000010:3.0:1788205234.332306:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647000. 00000100:00000010:2.0:1788205234.332313:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647180. 00010000:00000001:0.0:1788205234.844256:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205234.844265:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205234.844276:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205234.844290:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205234.844294:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844299:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205234.844301:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844304:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844306:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844311:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205234.844313:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205234.844314:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205234.844315:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844317:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205234.844318:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844320:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844322:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844325:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205234.844327:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205234.844328:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205234.844329:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844330:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205234.844332:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844334:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205234.844335:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205234.844581:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205234.844587:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205234.844590:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205234.844595:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205234.844599:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205234.844603:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205234.844606:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205234.844609:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205234.844611:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205234.844779:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205234.844786:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205234.844788:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205234.844792:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205234.844796:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205234.844799:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205234.844802:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205234.844804:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205234.844807:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205234.846841:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788205234.846849:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788205234.846853:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205235.868249:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205235.868254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205235.868260:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205235.868261:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205235.868265:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205235.868269:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205235.868271:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205235.868271:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205235.868276:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:1.0:1788205235.868278:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205235.868279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205235.868283:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205235.868284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205235.868287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205235.868288:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205235.868292:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205235.868296:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205235.868399:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205235.868403:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205235.868406:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205235.868456:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205235.868460:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205235.868462:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205235.868464:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205235.868467:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205235.868469:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205235.868472:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205235.868474:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205235.868476:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205236.892260:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205236.892270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205236.892279:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205236.892280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205236.892284:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205236.892285:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205236.892289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205236.892290:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205236.893636:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205236.893645:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205236.893647:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205236.893653:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205236.893657:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205236.893714:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205236.893718:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205236.893720:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205236.893723:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205236.894752:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205236.894757:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205236.894761:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205236.894770:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205236.894773:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205236.894774:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205236.894777:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205236.894779:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205236.894782:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205236.894788:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205236.894791:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205236.894793:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205237.916222:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205237.916230:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205237.916232:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205237.916237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205237.916237:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205237.916240:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205237.916241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205237.916243:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205237.916245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205237.916246:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205237.916247:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205237.916248:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205237.916250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205237.916251:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205237.916252:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205237.916254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205237.916255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205237.916326:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205237.916329:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205237.916330:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205237.916332:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205237.916334:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205237.916336:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205237.916339:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205237.916341:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205237.916343:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205237.916446:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205237.916450:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205237.916453:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205238.108201:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205238.108209:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205238.108211:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205238.108214:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205238.108218:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205238.108224:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26924 00000400:00000200:1.0:1788205238.108232:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205238.108237:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205238.108240:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205238.108242:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108244:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108245:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108247:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205238.108250:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205238.108251:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205238.108253:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205238.108256:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205238.108257:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205238.108258:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205238.108260:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108261:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108263:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108264:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205238.108266:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205238.108268:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108269:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205238.108271:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205238.108274:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205238.108276:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108277:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205238.108278:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205238.108288:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205238.108292:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205238.108295:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205238.108296:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108299:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205238.108301:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205238.108318:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205238.108327:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9000. 00000100:00000001:1.0:1788205238.108334:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205238.108342:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205238.108350:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205238.108354:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205238.108357:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205238.108374:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205238.108376:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205238.108379:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205238.108380:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205238.108381:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205238.108383:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205238.108385:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26924 00000400:00000200:1.0:1788205238.108387:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205238.108389:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205238.108391:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205238.108393:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108394:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108395:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108397:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205238.108398:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205238.108399:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205238.108400:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205238.108401:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205238.108402:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205238.108404:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205238.108406:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108408:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108410:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108413:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205238.108415:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205238.108418:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205238.108420:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205238.108421:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205238.108426:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205238.108427:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205238.108431:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788205238.108433:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205238.108436:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108441:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:3.0:1788205238.108441:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788205238.108447:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205238.108454:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.108456:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205238.108458:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.108459:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205238.108464:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000020:00000040:1.0:1788205238.108468:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205238.108470:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205238.108471:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108474:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205238.108475:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205238.108476:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205238.108478:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.108481:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1788205238.108484:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788205238.108485:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108490:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205238.108492:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543528064:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205238.108498:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1788205238.108500:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788205238.108503:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205238.108507:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.108512:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788205238.108513:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205238.108515:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205238.108518:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788205238.108519:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205238.108524:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108530:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205238.108533:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205238.108534:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:1.0:1788205238.108538:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205238.108540:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:3.0:1788205238.108541:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000100:00080000:1.0:1788205238.108542:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205238.108547:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205238.108549:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26924 00000400:00000200:1.0:1788205238.108552:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205238.108553:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000400:00000200:1.0:1788205238.108555:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205238.108558:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205238.108560:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108562:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1788205238.108562:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543528064, portal 25 00000100:00000040:1.0:1788205238.108566:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108568:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205238.108568:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205238.108571:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000020:00000001:1.0:1788205238.108572:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205238.108573:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205238.108576:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205238.108578:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205238.108580:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205238.108582:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205238.108583:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205238.108588:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108591:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108592:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108593:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205238.108595:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205238.108596:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108597:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205238.108598:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205238.108600:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205238.108601:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108602:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205238.108603:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205238.108605:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108606:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205238.108610:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000020:00000040:1.0:1788205238.108612:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205238.108613:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205238.108614:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108615:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205238.108616:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205238.108617:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205238.108620:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bbc00. 00000100:00000001:1.0:1788205238.108622:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205238.108625:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205238.108629:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205238.108631:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205238.108633:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205238.108637:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205238.108638:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205238.108640:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205238.108641:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205238.108642:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205238.108643:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205238.108645:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26924 00000400:00000200:1.0:1788205238.108647:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205238.108648:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205238.108650:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205238.108651:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108652:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108653:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108654:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205238.108656:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205238.108657:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205238.108658:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205238.108659:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205238.108660:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205238.108661:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205238.108663:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108756:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108759:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108760:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205238.108763:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205238.108765:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108768:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205238.108769:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205238.108772:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205238.108774:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108776:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205238.108778:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205238.108780:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108782:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205238.108785:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205238.108788:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205238.108790:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205238.108791:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108793:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205238.108794:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205238.108796:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205238.108800:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba400. 00000100:00000001:1.0:1788205238.108803:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205238.108808:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205238.108815:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205238.108817:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205238.108820:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205238.108825:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205238.108826:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205238.108828:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205238.108830:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205238.108831:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205238.108833:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205238.108835:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26924 00000400:00000200:1.0:1788205238.108838:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205238.108841:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205238.108843:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205238.108845:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108846:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108848:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108849:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205238.108850:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205238.108851:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205238.108852:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205238.108853:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205238.108854:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205238.108856:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205238.108858:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108860:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205238.108862:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108864:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205238.108866:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205238.108868:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205238.108870:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205238.108872:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205238.108874:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205238.108876:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108877:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205238.108878:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205238.108879:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108880:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205238.108891:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205238.108893:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205238.108894:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205238.108895:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205238.108896:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205238.108897:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205238.108898:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205238.108901:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba000. 00000100:00000001:1.0:1788205238.108913:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205238.108916:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205238.108921:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205238.108922:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205238.108924:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205238.108928:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205238.108932:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26934) 00000100:00000001:3.0:1788205238.109287:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205238.109291:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205238.109295:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734c38. 00000100:00000200:3.0:1788205238.109299:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543528064, offset 0 00000400:00000200:3.0:1788205238.109305:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205238.109312:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.109319:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205238.109323:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205238.109327:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205238.109330:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205238.109333:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205238.109337:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471474:471474:256:4294967295] 192.168.201.118@tcp LPNI seq info [471333:471333:8:4294967295] 00000400:00000200:3.0:1788205238.109341:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.109350:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205238.109355:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205238.109360:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdc00. 00000800:00000200:3.0:1788205238.109367:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205238.109578:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205238.109602:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205238.109607:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205238.109610:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321598214 00000100:00000010:3.0:1788205238.109616:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcb40. 00000100:00000001:3.0:1788205238.109623:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205238.109628:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321598214 00000100:00000010:3.0:1788205238.109707:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc480. 00000100:00000001:3.0:1788205238.109710:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205238.109715:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205238.109717:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.109720:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.109723:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205238.109730:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205238.109732:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788205238.109735:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205238.109736:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205238.109738:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205238.109743:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205238.109751:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205238.109752:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.109754:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205238.109756:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205238.109759:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205238.109762:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528192:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205238.109766:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205238.109769:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205238.109771:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.109772:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205238.109777:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205238.109779:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205238.109839:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205238.109844:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77420. 00000400:00000010:3.0:1788205238.109849:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734bb0. 00000100:00000200:3.0:1788205238.109853:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543528192, portal 10 00000100:00000001:3.0:1788205238.109855:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205238.109856:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788205238.109864:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205238.109871:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205238.109874:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205238.109877:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734d48. 00000100:00000200:3.0:1788205238.109879:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543528192, offset 0 00000400:00000200:3.0:1788205238.109884:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205238.109888:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.109892:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205238.109895:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205238.109898:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205238.109900:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205238.109902:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205238.109905:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471475:471475:255:4294967295] 192.168.201.118@tcp LPNI seq info [471334:471334:7:4294967295] 00000400:00000200:3.0:1788205238.109909:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.109917:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205238.109921:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205238.109925:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdb00. 00000800:00000200:3.0:1788205238.109929:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205238.109933:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205238.109935:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205238.109939:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205238.109942:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205238.109943:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321596947 00000100:00000001:3.0:1788205238.109946:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321598214 00000100:00000001:3.0:1788205238.109948:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321596947 00000100:00000001:3.0:1788205238.109950:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321598214 00000100:00000001:3.0:1788205238.109952:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205238.109953:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.109955:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205238.109960:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205238.109966:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205238.109968:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.109970:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205238.109971:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205238.109973:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205238.110462:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528320:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205238.110474:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205238.110477:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205238.110485:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.110487:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205238.110492:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205238.110494:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205238.110497:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205238.110500:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb771b8. 00000400:00000010:3.0:1788205238.110504:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734550. 00000100:00000200:3.0:1788205238.110508:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543528320, portal 10 00000100:00000001:3.0:1788205238.110510:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205238.110511:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205238.110518:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205238.110525:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205238.110528:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205238.110530:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734660. 00000100:00000200:3.0:1788205238.110533:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543528320, offset 0 00000400:00000200:3.0:1788205238.110537:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205238.110540:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.110544:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205238.110547:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205238.110549:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205238.110552:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205238.110554:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205238.110557:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471476:471476:254:4294967295] 192.168.201.118@tcp LPNI seq info [471335:471335:6:4294967295] 00000400:00000200:3.0:1788205238.110561:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.110567:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205238.110571:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205238.110574:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bdd00. 00000800:00000200:3.0:1788205238.110578:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205238.110582:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205238.110585:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205238.110588:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205238.110591:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205238.110592:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321596949 00000100:00000001:3.0:1788205238.110595:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321598214 00000100:00000001:3.0:1788205238.110596:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321596949 00000100:00000001:3.0:1788205238.110598:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321598214 00000100:00000001:3.0:1788205238.110600:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205238.110602:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.110604:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205238.110608:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205238.110615:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205238.110616:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.110619:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205238.110620:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205238.110622:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205238.110625:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528448:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205238.110628:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205238.110630:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205238.110632:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.110645:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205238.110649:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205238.110651:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205238.110654:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205238.110656:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77a50. 00000400:00000010:3.0:1788205238.110660:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b987343b8. 00000100:00000200:3.0:1788205238.110662:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543528448, portal 4 00000100:00000001:3.0:1788205238.110672:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205238.110673:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205238.110679:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205238.110685:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205238.110687:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205238.110689:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734f68. 00000100:00000200:3.0:1788205238.110691:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543528448, offset 0 00000400:00000200:3.0:1788205238.110695:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205238.110699:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.110702:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205238.110704:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205238.110707:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205238.110709:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205238.110711:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205238.110714:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471477:471477:253:4294967295] 192.168.201.118@tcp LPNI seq info [471336:471336:5:4294967295] 00000400:00000200:3.0:1788205238.110717:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.110723:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205238.110727:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205238.110730:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd700. 00000800:00000200:3.0:1788205238.110733:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205238.110736:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205238.110738:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205238.110741:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205238.110743:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205238.110745:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321596958 00000100:00000001:3.0:1788205238.110747:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321598214 00000100:00000001:3.0:1788205238.110750:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321596958 00000100:00000001:3.0:1788205238.110751:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321598214 00000100:00000001:3.0:1788205238.110753:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205238.110755:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.110757:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205238.110760:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205238.110766:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205238.110767:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.110769:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205238.110770:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205238.110772:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205238.110774:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528576:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205238.110777:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205238.110779:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205238.110780:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205238.110782:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205238.110784:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205238.110797:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205238.110800:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205238.110802:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77210. 00000400:00000010:3.0:1788205238.110805:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205238.110807:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543528576, portal 4 00000100:00000001:3.0:1788205238.110809:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205238.110811:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205238.110815:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205238.110821:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205238.110823:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205238.110825:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734330. 00000100:00000200:3.0:1788205238.110828:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543528576, offset 0 00000400:00000200:3.0:1788205238.110838:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205238.110842:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.110845:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205238.110847:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205238.110850:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205238.110852:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205238.110854:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205238.110857:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471478:471478:252:4294967295] 192.168.201.118@tcp LPNI seq info [471337:471337:4:4294967295] 00000400:00000200:3.0:1788205238.110860:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205238.110865:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205238.110869:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205238.110872:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bd900. 00000800:00000200:3.0:1788205238.110876:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205238.110879:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205238.110882:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205238.110885:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205238.110888:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205238.110889:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321596981 00000100:00000001:3.0:1788205238.110892:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321598214 00000100:00000001:3.0:1788205238.110894:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321596981 00000100:00000001:3.0:1788205238.110896:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321598214 00000100:00000001:3.0:1788205238.110898:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205238.110899:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.110902:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.110904:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205238.110908:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205238.110910:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788205238.110912:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205238.110913:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205238.110915:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.110917:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205238.940269:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205238.940280:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205238.940282:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205238.940289:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205238.940294:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205238.940299:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205238.940303:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205238.940306:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205238.940310:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205238.940411:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205238.940417:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205238.940418:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205238.940421:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205238.940424:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205238.940426:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205238.940428:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205238.940431:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205238.940433:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205238.940534:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205238.940539:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205238.940545:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205238.940546:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205238.940550:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205238.940551:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205238.940555:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205238.940556:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205238.940628:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205238.940632:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205238.940635:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205239.452257:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321598214, at 4321598272 00000100:00000010:2.0:1788205239.452281:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcb40. 00000100:00000040:3.0:1788205239.452916:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321598214, at 4321598272 00000100:00000010:3.0:1788205239.452930:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc480. 00010000:00000001:0.0:1788205239.964797:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205239.964806:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205239.964809:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205239.964813:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205239.964819:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205239.964821:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205239.964823:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205239.964825:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205239.964830:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205239.964831:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205239.964835:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205239.964836:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205239.964839:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205239.964840:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205239.965814:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205239.965823:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205239.965826:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205239.965832:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205239.965838:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205239.965843:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205239.965847:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205239.965850:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205239.965854:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205239.965896:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205239.965899:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205239.965901:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205239.965903:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205239.965905:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205239.965907:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205239.965909:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205239.965912:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205239.965914:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205239.966000:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205239.966005:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205239.966009:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205240.988335:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205240.988341:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205240.988347:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205240.988349:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205240.988352:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205240.988362:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205240.988368:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205240.988372:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205240.988375:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205240.988378:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205240.988382:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205240.988474:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205240.988479:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205240.988480:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205240.988483:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205240.988485:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205240.988487:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205240.988490:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205240.988493:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205240.988495:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205240.989342:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205240.989346:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205240.989352:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205240.989353:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205240.989356:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205240.989357:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205240.989832:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205240.989836:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205240.989839:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788205241.181252:0:256122:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.118:988 00000400:00000100:2.0:1788205241.181310:0:256122:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.118@tcp at host 192.168.201.118 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1788205241.181321:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205241.181325:0:256122:0:(router.c:1798:lnet_notify()) 192.168.201.18@tcp notifying 192.168.201.118@tcp: down 00000400:00000200:2.0:1788205241.181332:0:256122:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:2.0:1788205241.181337:0:256122:0:(router.c:1895:lnet_notify()) Old news 00000800:00000100:2.0:1788205241.181344:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205241.181348:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdc00. 00000400:00000200:2.0:1788205241.181356:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205241.181360:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205241.181368:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734c38 00000400:00000010:2.0:1788205241.181670:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734c38. 00000100:00000001:2.0:1788205241.181677:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205241.181688:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205241.181716:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205241.181720:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205241.181728:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205241.181730:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205241.181743:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205241.181746:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdb00. 00000400:00000200:2.0:1788205241.181751:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205241.181755:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205241.181759:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734d48 00000400:00000010:2.0:1788205241.181762:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734d48. 00000100:00000001:2.0:1788205241.181764:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205241.181770:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.181778:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788205241.181780:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205241.181788:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.181790:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205241.181795:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788205241.181801:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205241.181803:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1788205241.181811:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205238/real 1788205241] req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1788205241.181812:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205241.181821:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bdd00. 00000400:00000200:2.0:1788205241.181830:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205241.181845:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000001:3.0:1788205241.181849:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205241.181852:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205241.181856:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77420. 00000400:00000200:3.0:1788205241.181862:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734bb0 00000400:00000010:3.0:1788205241.181864:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734bb0. 00000100:00000001:3.0:1788205241.181868:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1788205241.181869:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734660 00000400:00000010:2.0:1788205241.181875:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734660. 00000100:00000200:3.0:1788205241.181876:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205241.181883:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205241.181892:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205241.181894:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205241.181933:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205241.181945:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.181947:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205241.181952:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.181960:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788205241.181967:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205241.181969:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205241.181975:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000100:2.0:1788205241.181977:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205241.181982:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd700. 00000100:00100000:3.0:1788205241.181986:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788205241.181989:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205241.182008:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205241.182012:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734f68 00000400:00000010:2.0:1788205241.182015:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734f68. 00000100:00000001:2.0:1788205241.182018:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205241.182023:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205241.182032:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205241.182036:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205241.182041:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205241.182043:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1788205241.182048:0:256122:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.118@tcp 00000800:00000010:2.0:1788205241.182050:0:256122:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bd900. 00000400:00000200:2.0:1788205241.182053:0:256122:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1788205241.182056:0:256122:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1788205241.182059:0:256122:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734330 00000400:00000010:2.0:1788205241.182061:0:256122:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734330. 00000100:00000001:2.0:1788205241.182064:0:256122:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1788205241.182068:0:256122:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205241.182075:0:256122:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1788205241.182079:0:256122:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205263 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788205241.182085:0:256122:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205241.182087:0:256122:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205241.182166:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205241.182174:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182182:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205241.182186:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182188:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182190:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182193:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205241.182196:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205241.182200:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205241.182205:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205241.182208:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205241.182212:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205241.182220:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528192:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205241.182225:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205241.182229:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205238/real 1788205241] req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205241.182237:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205241.182239:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205241.182241:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb771b8. 00000400:00000200:3.0:1788205241.182244:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734550 00000400:00000010:3.0:1788205241.182246:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734550. 00000100:00000001:3.0:1788205241.182248:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205241.182251:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205241.182259:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182264:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205241.182266:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182267:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182268:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205241.182272:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182278:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205241.182281:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182287:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205241.182289:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182290:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182291:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182293:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205241.182295:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205241.182297:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205241.182300:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205241.182302:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205241.182305:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205241.182311:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528320:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205241.182316:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205241.182320:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205238/real 1788205241] req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205241.182326:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205241.182328:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205241.182329:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77a50. 00000400:00000200:3.0:1788205241.182331:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b987343b8 00000400:00000010:3.0:1788205241.182333:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b987343b8. 00000100:00000001:3.0:1788205241.182335:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205241.182338:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205241.182345:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182350:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205241.182352:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182353:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182354:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205241.182358:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182363:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205241.182366:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182372:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205241.182374:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182375:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182376:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182377:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205241.182379:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205241.182382:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205241.182385:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205241.182387:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205241.182391:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205241.182398:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528448:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205241.182402:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205241.182406:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205238/real 1788205241] req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205241.182424:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205241.182426:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205241.182427:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77210. 00000400:00000200:3.0:1788205241.182430:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205241.182431:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205241.182433:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205241.182436:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205241.182444:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182450:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205241.182452:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182454:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182455:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205241.182459:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182464:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205241.182468:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182535:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205241.182537:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182539:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182540:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182541:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205241.182543:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205241.182546:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205241.182549:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205241.182551:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205241.182555:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205241.182561:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528576:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205241.182565:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205241.182569:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205238/real 1788205241] req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205241.182576:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205241.182577:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205241.182579:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205241.182591:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205241.182592:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205241.182595:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205241.182598:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205241.182621:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182627:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205241.182628:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182630:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182632:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205241.182635:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182640:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205241.182643:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182649:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205241.182651:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182652:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205241.182653:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182655:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205241.182656:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205241.182659:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205241.182662:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205241.182664:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205241.182667:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205241.182674:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543528064:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205241.182678:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182681:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205241.182683:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543528192/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182690:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205241.182692:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205241.182696:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205241.182700:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205241.182703:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205241.182705:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205241.182708:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205241.182710:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788205241.182713:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205241.182715:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205241.182717:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205241.182723:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205241.182724:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182726:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205241.182729:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543528320/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182734:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205241.182735:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205241.182737:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205241.182739:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205241.182740:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205241.182742:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205241.182744:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205241.182745:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:3.0:1788205241.182747:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205241.182748:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205241.182749:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205241.182754:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205241.182755:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182757:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205241.182759:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543528448/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182765:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205241.182766:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205241.182767:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205241.182769:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205241.182771:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205241.182772:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205241.182774:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205241.182775:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:3.0:1788205241.182777:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205241.182779:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205241.182780:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205241.182782:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205241.182783:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182784:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205241.182788:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543528576/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182793:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205241.182794:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205241.182795:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205241.182798:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205241.182799:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205241.182800:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205241.182802:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205241.182804:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba000. 02000000:00000001:3.0:1788205241.182806:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205241.182807:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205241.182808:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205241.182812:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205241.182814:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182815:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205241.182819:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543528064/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205263 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205241.182824:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205241.182825:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205241.182827:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205241.182829:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205241.182830:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205241.182831:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205241.182833:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205241.182834:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:3.0:1788205241.182837:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205241.182838:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205241.182839:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205241.182841:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205241.182842:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182844:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205241.182853:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205241.182855:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205241.182857:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205241.182859:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205242.012204:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205242.012211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205242.012216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205242.012217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205242.012221:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205242.012228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205242.012230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205242.012233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205242.012234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205242.012240:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205242.012240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205242.012243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205242.012246:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205242.012249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205242.012250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205242.012252:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205242.012256:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205242.012435:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205242.012440:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205242.012442:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205242.012444:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205242.012446:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205242.012449:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205242.012451:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205242.012454:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205242.012456:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205242.012527:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205242.012532:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205242.012535:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205243.036268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205243.036279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205243.036287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205243.036288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205243.036292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205243.036293:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205243.036297:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205243.036298:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.036429:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205243.036436:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205243.036438:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.036442:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205243.036447:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205243.036451:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.036454:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.036458:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.036461:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205243.036641:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205243.036647:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205243.036651:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205243.036786:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205243.036792:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205243.036794:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205243.036797:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205243.036799:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205243.036802:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205243.036804:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205243.036807:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205243.036809:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205243.228399:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205243.228416:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205243.228419:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205243.228425:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205243.228431:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205243.228441:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26929 00000400:00000200:1.0:1788205243.228455:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205243.228466:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205243.228470:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205243.228474:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228478:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.228481:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228484:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205243.228489:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205243.228491:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205243.228495:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205243.228498:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205243.228501:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205243.228504:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205243.228506:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228508:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.228510:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228512:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205243.228516:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205243.228520:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228522:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205243.228525:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205243.228530:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205243.228533:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.228537:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205243.228539:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205243.228551:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205243.228555:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205243.228560:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205243.228562:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.228566:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205243.228570:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205243.228574:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205243.228621:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9000. 00000100:00000001:1.0:1788205243.228630:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205243.228641:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205243.228652:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205243.228658:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205243.228662:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205243.228694:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205243.228698:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205243.228702:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205243.228703:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205243.228705:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205243.228708:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205243.228711:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26929 00000400:00000200:1.0:1788205243.228715:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205243.228719:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205243.228721:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205243.228724:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228726:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.228729:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228731:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205243.228734:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205243.228735:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205243.228736:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205243.228739:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205243.228741:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205243.228743:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205243.228746:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228748:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.228750:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228752:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205243.228754:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205243.228758:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228760:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205243.228762:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205243.228769:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205243.228771:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.228772:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205243.228777:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205243.228780:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.228783:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205243.228787:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788205243.228789:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205243.228792:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205243.228793:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.228795:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205243.228796:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205243.228798:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205243.228801:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba000. 00000100:00000001:1.0:1788205243.228805:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205243.228810:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205243.228819:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205243.228821:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205243.228825:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205243.228833:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205243.228849:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205243.228853:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205243.228855:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205243.228857:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205243.228860:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205243.228862:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26929 00000400:00000200:1.0:1788205243.228866:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205243.228870:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205243.228872:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205243.228875:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228877:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.228879:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228881:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205243.228883:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205243.228885:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205243.228886:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205243.228888:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205243.228890:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205243.228892:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205243.228894:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228896:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.228907:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228910:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205243.228912:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205243.228915:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205243.228918:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205243.228919:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205243.228922:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205243.228924:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.228926:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205243.228928:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205243.228930:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.228932:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205243.228936:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205243.228939:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205243.228951:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205243.228953:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.228955:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205243.228957:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205243.228958:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205243.228963:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba400. 00000100:00000001:1.0:1788205243.228967:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205243.228972:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205243.228979:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205243.228981:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205243.228985:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205243.228992:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205243.228996:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205243.228999:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205243.229000:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205243.229002:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205243.229005:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205243.229007:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26929 00000400:00000200:1.0:1788205243.229012:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205243.229016:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205243.229018:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205243.229021:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229023:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.229025:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229027:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205243.229029:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205243.229030:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205243.229032:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205243.229034:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205243.229036:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205243.229038:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205243.229041:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229043:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.229045:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229046:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205243.229049:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205243.229051:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229054:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205243.229055:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205243.229058:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205243.229060:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.229062:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205243.229064:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205243.229066:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.229068:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205243.229073:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205243.229076:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205243.229078:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205243.229079:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.229081:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205243.229083:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205243.229085:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205243.229088:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bbc00. 00000100:00000001:1.0:1788205243.229125:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205243.229130:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205243.229138:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205243.229141:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205243.229144:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205243.229153:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205243.229155:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205243.229159:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205243.229160:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205243.229161:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205243.229164:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205243.229167:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26929 00000400:00000200:1.0:1788205243.229183:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205243.229186:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205243.229189:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205243.229192:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229194:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.229196:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229198:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205243.229201:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205243.229202:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205243.229204:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205243.229205:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205243.229208:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205243.229210:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205243.229215:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229217:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205243.229219:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229221:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205243.229223:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205243.229227:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205243.229229:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205243.229231:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205243.229234:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205243.229236:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.229238:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205243.229240:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205243.229243:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.229244:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205243.229249:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205243.229251:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205243.229253:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205243.229255:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205243.229256:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205243.229258:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205243.229259:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205243.229263:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb800. 00000100:00000001:1.0:1788205243.229267:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205243.229272:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205243.229280:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205243.229282:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205243.229286:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205243.229293:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205243.229299:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26939) 00000100:00000001:3.0:1788205243.233222:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205243.233239:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205243.233244:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205243.233266:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233284:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.233286:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.233293:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205243.233300:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205243.233303:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.233308:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543528704:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205243.233313:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205243.233322:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205243.233329:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.233332:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205243.233346:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205243.233348:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205243.233359:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205243.233364:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205243.233375:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205243.233387:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543528704, portal 25 00000100:00000001:3.0:1788205243.233389:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205243.233391:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788205243.233398:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233405:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205243.233408:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205243.233410:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205243.233414:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543528704, offset 0 00000400:00000200:3.0:1788205243.233421:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205243.233435:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.233449:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205243.233458:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205243.233463:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205243.233469:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205243.233472:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205243.233478:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471479:471479:256:4294967295] 192.168.201.118@tcp LPNI seq info [471338:471338:8:4294967295] 00000400:00000200:3.0:1788205243.233483:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.233494:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205243.233503:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.233510:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205243.233521:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205243.233525:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788205243.233533:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205243.233538:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788205243.233543:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.233547:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205243.233552:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205243.233557:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205243.233565:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205243.233568:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205243.233574:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205243.233588:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233601:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.233605:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233612:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.233614:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205243.233623:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205243.233627:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205243.233631:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321603338 00000100:00000010:3.0:1788205243.233639:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc480. 00000100:00000001:3.0:1788205243.233649:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205243.233658:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321603338 00000100:00000010:3.0:1788205243.233662:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc600. 00000100:00000001:3.0:1788205243.233666:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205243.233669:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205243.233673:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.233675:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205243.233680:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205243/real 1788205243] req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205243.233692:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205243.233694:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205243.233697:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205243.233699:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205243.233701:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205243.233704:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205243.233708:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205243.233717:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233724:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205243.233726:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.233728:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.233731:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205243.233735:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233742:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205243.233749:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233757:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205243.233760:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.233762:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.233763:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.233769:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205243.233772:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205243.233777:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205243.233783:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205243.233787:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205243.233793:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205243.233802:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543528704:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205243.233808:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205243.233813:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233826:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.233828:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.233831:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205243.233834:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205243.233837:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.233840:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528832:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205243.233844:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205243.233848:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205243.233850:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.233851:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205243.233857:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205243.233865:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205243.233869:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205243.233873:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205243.233878:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205243.233882:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543528832, portal 10 00000100:00000001:3.0:1788205243.233884:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205243.233886:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000040:3.0:1788205243.233894:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.233902:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205243.233904:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205243.233906:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205243.233909:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543528832, offset 0 00000400:00000200:3.0:1788205243.233913:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205243.233917:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.233922:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205243.233925:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205243.233928:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205243.233931:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205243.233934:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205243.233937:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471480:471480:256:4294967295] 192.168.201.118@tcp LPNI seq info [471339:471339:8:4294967295] 00000400:00000200:3.0:1788205243.233942:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.233948:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205243.233953:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.233956:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205243.233961:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205243.233964:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788205243.233968:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205243.233971:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788205243.233976:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.233978:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205243.233981:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205243.233985:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205243.233989:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205243.233990:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205243.233993:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205243.233996:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234005:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.234008:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234422:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.234424:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205243.234432:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205243.234438:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205243.234442:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205243.234445:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321602072 00000100:00000001:3.0:1788205243.234449:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321603338 00000100:00000001:3.0:1788205243.234451:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321602072 00000100:00000001:3.0:1788205243.234454:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321603338 00000100:00000001:3.0:1788205243.234456:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205243.234460:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.234463:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205243.234468:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205243/real 1788205243] req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205243.234477:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205243.234479:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205243.234481:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205243.234484:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205243.234485:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205243.234488:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205243.234491:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205243.234499:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234506:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205243.234507:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.234510:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.234512:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205243.234515:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234522:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205243.234526:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234532:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205243.234535:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.234536:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.234538:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.234540:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205243.234543:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205243.234546:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205243.234551:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205243.234553:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205243.234557:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205243.234564:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528832:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205243.234569:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205243.234574:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234593:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.234595:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.234598:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205243.234600:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205243.234602:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.234606:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528960:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205243.234609:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205243.234612:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205243.234614:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.234616:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205243.234621:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205243.234623:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205243.234626:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205243.234629:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205243.234633:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205243.234637:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543528960, portal 10 00000100:00000001:3.0:1788205243.234639:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205243.234641:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205243.234647:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234653:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205243.234656:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205243.234658:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205243.234661:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543528960, offset 0 00000400:00000200:3.0:1788205243.234666:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205243.234670:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.234675:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205243.234679:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205243.234681:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205243.234684:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205243.234687:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205243.234690:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471481:471481:256:4294967295] 192.168.201.118@tcp LPNI seq info [471340:471340:8:4294967295] 00000400:00000200:3.0:1788205243.234694:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.234700:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205243.234704:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.234708:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205243.234712:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205243.234716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788205243.234721:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205243.234723:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788205243.234728:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.234730:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205243.234734:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205243.234738:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205243.234742:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205243.234744:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205243.234746:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205243.234750:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234758:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.234761:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234767:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.234769:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205243.234773:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205243.234775:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205243.234779:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205243.234781:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321602073 00000100:00000001:3.0:1788205243.234783:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321603338 00000100:00000001:3.0:1788205243.234786:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321602073 00000100:00000001:3.0:1788205243.234788:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321603338 00000100:00000001:3.0:1788205243.234790:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205243.234792:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.234794:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205243.234797:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205243/real 1788205243] req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205243.234803:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205243.234805:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205243.234806:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205243.234808:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205243.234810:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205243.234812:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205243.234815:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205243.234822:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234828:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205243.234830:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.234831:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.234833:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205243.234836:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234842:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205243.234847:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.234853:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205243.234856:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.234857:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.234858:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.234860:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205243.234863:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205243.234865:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205243.234869:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205243.234871:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205243.234874:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205243.234881:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543528960:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205243.234885:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205243.234889:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.237313:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.237317:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.237324:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205243.237329:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205243.237333:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.237343:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529088:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205243.237350:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205243.237356:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205243.237359:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.237363:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205243.237375:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205243.237378:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205243.237389:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205243.237528:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205243.237551:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205243.237561:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543529088, portal 4 00000100:00000001:3.0:1788205243.237564:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205243.237565:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205243.237589:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.237598:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205243.237602:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205243.237605:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205243.237608:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543529088, offset 0 00000400:00000200:3.0:1788205243.237617:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205243.237629:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.237641:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205243.237647:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205243.237653:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205243.237657:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205243.237660:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205243.237665:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471482:471482:256:4294967295] 192.168.201.118@tcp LPNI seq info [471341:471341:8:4294967295] 00000400:00000200:3.0:1788205243.237670:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.237681:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205243.237688:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.237694:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205243.237703:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205243.237707:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788205243.237715:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205243.237720:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788205243.237726:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.237729:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205243.237736:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205243.237742:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205243.237750:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205243.240647:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205243.240663:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205243.240678:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.240693:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.240697:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.240705:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.240707:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205243.240721:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205243.240727:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205243.240734:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205243.240738:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321602088 00000100:00000001:3.0:1788205243.240743:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321603338 00000100:00000001:3.0:1788205243.240746:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321602088 00000100:00000001:3.0:1788205243.240749:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321603338 00000100:00000001:3.0:1788205243.240752:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205243.241339:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.241344:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205243.241351:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205243/real 1788205243] req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205243.241366:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205243.241368:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205243.241372:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205243.241376:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205243.241379:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205243.241382:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205243.241386:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205243.241396:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.241657:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205243.241659:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.241662:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.241664:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205243.241669:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.241676:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205243.241680:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.241688:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205243.241692:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.241693:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.241695:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.241698:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205243.241700:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205243.241704:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205243.241709:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205243.241712:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205243.241717:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205243.241726:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529088:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205243.241732:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205243.241735:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.241741:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205243.241743:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.241748:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205243.241751:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205243.241754:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205243.241757:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529216:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205243.241771:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205243.241775:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205243.241778:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205243.241780:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205243.241790:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205243.241793:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205243.241798:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205243.241802:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205243.242011:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205243.242019:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543529216, portal 4 00000100:00000001:3.0:1788205243.242022:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205243.242024:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205243.242032:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242038:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205243.242041:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205243.242044:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205243.242047:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543529216, offset 0 00000400:00000200:3.0:1788205243.242052:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205243.242061:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.242070:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205243.242076:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205243.242080:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205243.242085:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205243.242107:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205243.242112:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471483:471483:256:4294967295] 192.168.201.118@tcp LPNI seq info [471342:471342:8:4294967295] 00000400:00000200:3.0:1788205243.242118:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205243.242129:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205243.242136:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.242142:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205243.242148:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205243.242152:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000200:3.1:1788205243.242159:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205243.242162:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 58 secs later) 00000800:00000100:3.0:1788205243.242167:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205243.242170:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205243.242175:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205243.242180:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205243.242185:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205243.242187:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205243.242189:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205243.242193:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242202:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.242205:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205268 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242211:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.242213:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205243.242218:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205243.242221:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205243.242225:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205243.242227:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321602113 00000100:00000001:3.0:1788205243.242229:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321603338 00000100:00000001:3.0:1788205243.242232:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321602113 00000100:00000001:3.0:1788205243.242234:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321603338 00000100:00000001:3.0:1788205243.242236:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205243.242238:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.242240:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205243.242243:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205243/real 1788205243] req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205243.242252:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205243.242253:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205243.242255:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205243.242257:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205243.242259:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205243.242261:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205243.242263:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205243.242272:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242278:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205243.242279:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242281:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.242282:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205243.242286:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242292:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205243.242296:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242302:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205243.242304:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242306:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205243.242308:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242310:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205243.242311:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205243.242314:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205243.242317:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205243.242320:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205243.242323:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205243.242330:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529216:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205243.242334:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242337:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.242341:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543528704/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242348:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205243.242351:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205243.242354:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:3.0:1788205243.242357:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205243.242360:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205243.242362:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205243.242364:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205243.242367:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:3.0:1788205243.242371:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205243.242373:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205243.242375:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788205243.242381:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205243.242382:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242385:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.242389:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543528832/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242394:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205243.242396:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205243.242397:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:3.0:1788205243.242519:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205243.242522:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205243.242523:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205243.242525:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205243.242527:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba000. 02000000:00000001:3.0:1788205243.242530:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205243.242531:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205243.242533:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205243.242540:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205243.242541:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242544:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.242550:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543528960/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242557:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205243.242558:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205243.242560:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:3.0:1788205243.242563:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205243.242564:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205243.242566:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205243.242568:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205243.242569:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:3.0:1788205243.242571:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205243.242573:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205243.242574:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205243.242618:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205243.242620:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242621:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.242625:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543529088/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242632:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205243.242634:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205243.242636:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:3.0:1788205243.242638:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205243.242639:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205243.242640:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205243.242642:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205243.242643:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:3.0:1788205243.242645:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205243.242646:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205243.242648:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205243.242650:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205243.242651:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242653:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205243.242656:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543529216/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205268 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205243.242663:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205243.242664:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205243.242666:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412400. 02000000:00000001:3.0:1788205243.242668:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205243.242669:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205243.242670:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205243.242672:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205243.242673:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788205243.242675:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205243.242676:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205243.242678:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205243.242685:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205243.242686:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242689:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205243.242698:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205243.242700:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205243.242702:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205243.242704:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205244.060216:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205244.060225:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205244.060227:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205244.060230:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205244.060233:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205244.060235:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205244.060239:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205244.060242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205244.060244:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205244.060245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205244.060249:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205244.060252:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205244.060255:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205244.060355:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205244.060355:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205244.060359:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205244.060361:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205244.060364:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205244.060366:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205244.060368:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205244.060369:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205244.060373:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205244.060376:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205244.060378:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205244.060380:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205244.060382:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205244.060529:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205244.060533:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205244.060536:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205244.572154:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321603338, at 4321603392 00000100:00000010:2.0:1788205244.572167:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc480. 00000100:00000040:3.0:1788205244.572246:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321603338, at 4321603392 00000100:00000010:3.0:1788205244.572254:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc600. 00000100:00000001:3.0:1788205245.084219:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205245.084224:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205245.084231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205245.084231:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205245.084235:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205245.084241:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205245.084243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205245.084249:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205245.084250:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205245.084255:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205245.084257:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205245.084260:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205245.084262:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205245.084266:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788205245.084268:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205245.084270:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205245.084274:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205245.084274:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205245.084281:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205245.084286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205245.084287:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205245.084289:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205245.084290:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205245.084292:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205245.084293:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205245.084295:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205245.084297:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205245.084302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205245.084304:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205245.084305:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205245.084310:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205245.084310:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205245.084314:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205245.084317:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205245.084321:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205245.084322:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205245.084324:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205245.084326:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205245.084334:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205245.084338:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205245.084341:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205245.084344:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205245.084348:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205245.084351:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205245.084409:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205245.084414:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205245.084418:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205246.108357:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205246.108369:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205246.108371:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205246.108379:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205246.108384:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205246.108390:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205246.109146:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205246.109193:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205246.109196:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205246.109202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205246.109206:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205246.109210:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205246.109213:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205246.109216:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205246.109218:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205246.111197:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205246.111203:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205246.111207:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205246.111583:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205246.111586:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205246.111589:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205246.111590:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205246.111592:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205246.111597:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205246.111599:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205246.111600:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205246.111669:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205246.111672:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205246.111674:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205247.132227:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205247.132235:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205247.132238:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205247.132242:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205247.132244:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205247.132250:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205247.132252:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205247.132257:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205247.132261:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205247.132264:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205247.132267:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205247.132274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205247.132278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205247.132282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205247.132284:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205247.132290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205247.132292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205247.132467:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205247.132472:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205247.132474:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205247.132478:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205247.132481:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205247.132484:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205247.132487:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205247.132490:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205247.132492:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205247.134006:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205247.134011:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205247.134016:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205248.156238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205248.156245:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205248.156245:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205248.156253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205248.156253:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205248.156253:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205248.156256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.156263:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205248.156264:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.156266:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205248.156267:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205248.156273:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205248.156273:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205248.156273:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205248.156284:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205248.156286:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205248.156290:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205248.156291:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205248.156293:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205248.156299:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.156301:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205248.156303:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205248.156307:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205248.156309:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205248.156312:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205248.156314:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205248.156374:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205248.156378:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205248.156382:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205248.348263:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205248.348276:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205248.348279:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205248.348284:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205248.348289:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205248.348298:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26934 00000400:00000200:3.0:1788205248.348310:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205248.348319:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205248.348322:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205248.348326:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.348329:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205248.348332:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.348335:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205248.348339:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205248.348341:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205248.348344:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205248.349605:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205248.349615:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205248.349620:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205248.349629:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.349634:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205248.349637:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.349639:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205248.349642:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205248.349646:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205248.349650:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205248.349653:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205248.349657:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205248.349660:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.349662:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205248.349665:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205248.349676:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788205248.349680:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205248.349683:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205248.349685:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.349688:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205248.349691:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205248.349695:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205248.349703:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412400. 00000100:00000001:3.0:1788205248.349710:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205248.349719:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205248.349743:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205248.349748:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205248.349754:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205248.349777:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205248.349781:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205248.349786:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205248.349789:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205248.349791:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205248.349795:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205248.349815:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788205248.349816:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26934 00000100:00000001:1.0:1788205248.349822:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205248.349826:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1788205248.349829:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:1.0:1788205248.349835:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205248.349839:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205248.349843:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205248.349849:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.349855:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205248.349860:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205248.349861:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205248.349864:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.349865:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:1.0:1788205248.349869:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1788205248.349870:0:409030:0:(genops.c:814:class_conn2export()) Process entered 02000000:00000001:1.0:1788205248.349873:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1788205248.349874:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 02000000:00000001:1.0:1788205248.349878:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205248.349879:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205248.349883:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00100000:1.0:1788205248.349885:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543529344:192.168.201.118@tcp:250:kworker.0 00000020:00000001:3.0:1788205248.349888:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:1.0:1788205248.349892:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1788205248.349893:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 02000000:00000001:1.0:1788205248.349897:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1788205248.349900:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205248.349901:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.349905:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788205248.349907:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205248.349910:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 02000000:00000010:1.0:1788205248.349913:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 00000100:00000001:3.0:1788205248.349913:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000001:1.0:1788205248.349918:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205248.349919:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000400:00000200:1.0:1788205248.349925:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205248.349929:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00080000:3.0:1788205248.349934:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:1.0:1788205248.349937:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000001:3.0:1788205248.349939:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205248.349943:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1788205248.349946:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543529344, portal 25 00000100:00000001:1.0:1788205248.349950:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788205248.349950:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205248.349953:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 02000000:00000001:3.0:1788205248.349956:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.349959:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:1.0:1788205248.349961:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788205248.349965:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205248.349971:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.349973:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788205248.349975:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205248.349981:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000100:00000040:1.0:1788205248.349985:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205248.349989:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205248.349994:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543529344, offset 0 00000400:00000200:1.0:1788205248.350000:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205248.350006:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205248.350013:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205248.350017:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205248.350021:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205248.350024:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205248.350026:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205248.350044:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471484:471484:256:4294967295] 192.168.201.118@tcp LPNI seq info [471343:471343:8:4294967295] 00000400:00000200:1.0:1788205248.350048:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205248.350063:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205248.350069:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.350073:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205248.350080:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205248.350083:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205248.350112:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000040:3.0:1788205248.350116:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000800:00000200:1.1:1788205248.350117:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 02000000:00000001:3.0:1788205248.350124:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000800:00000100:1.0:1788205248.350125:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205248.350127:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1788205248.350128:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00000001:3.0:1788205248.350132:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205248.350135:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1788205248.350138:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:3.0:1788205248.350138:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205248.350145:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 02000000:00000010:3.0:1788205248.350146:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413800. 00000400:00000200:1.0:1788205248.350152:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00000001:3.0:1788205248.350152:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205248.350156:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205248.350160:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205248.350165:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205248.350171:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205248.350179:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788205248.350185:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1788205248.350186:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205248.350190:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350200:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205248.350201:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.350203:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205248.350211:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.350214:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205248.350218:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205248.350221:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205248.350223:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321608455 00000100:00080000:3.0:1788205248.350227:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205248.350231:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000010:1.0:1788205248.350232:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647c60. 00000100:00080000:3.0:1788205248.350234:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205248.350238:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205248.350241:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00080000:3.0:1788205248.350242:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26934 00000400:00000200:3.0:1788205248.350249:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205248.350253:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205248.350255:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205248.350258:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205248.350261:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321608455 00000100:00000001:3.0:1788205248.350261:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205248.350264:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.350266:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205248.350271:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000010:1.0:1788205248.350273:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647420. 00000020:00000040:3.0:1788205248.350274:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205248.350278:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205248.350279:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000020:00000040:3.0:1788205248.350281:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000001:1.0:1788205248.350283:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788205248.350285:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000001:1.0:1788205248.350287:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350291:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205248.350296:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205248/real 1788205248] req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788205248.350300:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205248.350308:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.350310:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205248.350312:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.350315:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000001:1.0:1788205248.350320:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:3.0:1788205248.350320:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000001:1.0:1788205248.350323:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788205248.350326:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:1.0:1788205248.350327:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000100:00000001:3.0:1788205248.350332:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205248.350333:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205248.350337:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 02000000:00000001:3.0:1788205248.350338:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205248.350341:0:256132:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1788205248.350343:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1788205248.350347:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205248.350348:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.350353:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205248.350357:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1788205248.350361:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788205248.350362:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.350367:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205248.350373:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205248.350374:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205248.350378:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1788205248.350379:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000100:00000001:1.0:1788205248.350382:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788205248.350384:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205248.350385:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788205248.350387:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205248.350392:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205248.350392:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205248.350395:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205248.350398:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350404:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1788205248.350405:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412800. 00000100:00000040:1.0:1788205248.350410:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205248.350412:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350421:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1788205248.350422:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205248.350426:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.350428:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205248.350429:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.350431:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205248.350433:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205248.350438:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205248.350446:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205248.350450:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788205248.350450:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205248.350455:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205248.350458:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205248.350464:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205248.350483:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543529344:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205248.350484:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350501:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788205248.350501:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788205248.350504:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.350508:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543529344/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205248.350512:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205248.350515:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205248.350518:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1788205248.350519:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788205248.350521:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205248.350526:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 00000100:00000001:3.0:1788205248.350526:0:409030:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1788205248.350531:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205248.350533:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00080000:3.0:1788205248.350534:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26934 00000020:00000040:1.0:1788205248.350536:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205248.350539:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205248.350542:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412400. 00000400:00000200:3.0:1788205248.350542:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:1.0:1788205248.350547:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1788205248.350549:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 02000000:00000001:1.0:1788205248.350550:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205248.350552:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00080000:3.0:1788205248.350554:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205248.350556:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205248.350558:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.350560:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205248.350560:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.350565:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205248.350568:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205248.350570:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205248.350570:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205248.350573:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000001:1.0:1788205248.350574:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350577:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205248.350578:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1788205248.350578:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000100:00000001:1.0:1788205248.350581:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788205248.350582:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000040:1.0:1788205248.350585:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205248.350587:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205248.350591:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205248.350596:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000001:1.0:1788205248.350597:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205248.350600:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205248.350603:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:1.0:1788205248.350604:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205248.350607:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205248.350608:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788205248.350610:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205248.350613:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00100000:1.0:1788205248.350615:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529472:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205248.350616:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205248.350619:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1788205248.350621:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 02000000:00000001:1.0:1788205248.350622:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205248.350624:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.350627:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788205248.350628:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 02000000:00000010:1.0:1788205248.350630:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 00000100:00000001:3.0:1788205248.350632:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.350633:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205248.350637:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1788205248.350639:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205248.350642:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 02000000:00000001:3.0:1788205248.350642:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1788205248.350647:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 02000000:00000001:3.0:1788205248.350647:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205248.350651:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543529472, portal 10 00000100:00000001:3.0:1788205248.350652:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205248.350655:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00100000:3.0:1788205248.350656:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:1.0:1788205248.350657:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000008:00000001:3.0:1788205248.350662:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.350664:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205248.350668:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000020:00000040:3.0:1788205248.350670:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205248.350673:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1788205248.350676:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205248.350676:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.350680:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205248.350683:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205248.350686:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350687:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205248.350690:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000010:3.0:1788205248.350692:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410400. 00000400:00000010:1.0:1788205248.350693:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205248.350698:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543529472, offset 0 00000100:00000001:3.0:1788205248.350700:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205248.350703:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205248.350709:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:3.0:1788205248.350710:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205248.350713:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205248.350719:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205248.350723:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205248.350724:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205248.350726:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205248.350728:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471485:471485:256:4294967295] 192.168.201.118@tcp LPNI seq info [471344:471344:8:4294967295] 00000400:00000200:1.0:1788205248.350730:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205248.350735:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205248.350737:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:3.0:1788205248.350740:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1788205248.350743:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.350747:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00000040:3.0:1788205248.350750:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1788205248.350753:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205248.350757:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205248.350765:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205248.350769:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000100:00000001:3.0:1788205248.350770:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788205248.350772:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.350775:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000100:00080000:3.0:1788205248.350775:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205248.350780:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1788205248.350781:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:1.0:1788205248.350784:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788205248.350785:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1788205248.350787:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000100:00080000:3.0:1788205248.350788:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1788205248.350789:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205248.350794:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1788205248.350794:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1788205248.350800:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788205248.350800:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26934 00000400:00000200:3.0:1788205248.350809:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205248.350811:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.350814:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205248.350817:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205248.350822:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205248.350828:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.350833:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205248.350834:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205248.350837:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205248.350838:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205248.350841:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.350843:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:1.0:1788205248.350844:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000020:00000001:3.0:1788205248.350848:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000200:1.0:1788205248.350850:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:3.0:1788205248.350852:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000001:1.0:1788205248.350853:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321607188 00000020:00000001:3.0:1788205248.350856:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00000001:1.0:1788205248.350857:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321608455 00000100:00000001:1.0:1788205248.350860:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321607188 00000020:00000040:3.0:1788205248.350861:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788205248.350862:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321608455 00000100:00000001:1.0:1788205248.350866:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788205248.350866:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788205248.350870:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205248.350871:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:1.0:1788205248.350872:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205248.350879:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205248/real 1788205248] req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205248.350879:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205248.350884:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205248.350888:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000400:00000001:1.0:1788205248.350894:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788205248.350894:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000001:1.0:1788205248.350896:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:3.0:1788205248.350899:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000010:1.0:1788205248.350900:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205248.350903:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205248.350906:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00080000:3.0:1788205248.350906:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205248.350910:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1788205248.350911:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205248.350914:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205248.350915:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205248.350920:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1788205248.350922:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205248.350924:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.350928:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205248.350931:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1788205248.350932:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205248.350935:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788205248.350935:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350937:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205248.350940:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205248.350941:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205248.350946:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788205248.350948:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788205248.350953:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:1.0:1788205248.350955:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788205248.350958:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1788205248.350959:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205248.350961:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205248.350965:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205248.350968:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788205248.350969:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788205248.350971:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350972:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.350976:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000010:3.0:1788205248.350977:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411400. 00000100:00000001:1.0:1788205248.350978:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.350983:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205248.350985:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205248.350988:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205248.350991:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205248.350992:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.350993:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788205248.351000:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1788205248.351004:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205248.351019:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205248.351024:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205248.351025:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529472:192.168.201.118@tcp:38:kworker.0 00000100:00000040:3.0:1788205248.351031:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351032:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205248.351039:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205248.351044:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351052:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1788205248.351053:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26944) 00000100:00000001:1.0:1788205248.351055:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.351059:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205248.351061:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205248.351063:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205248.351065:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529600:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205248.351068:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205248.351070:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205248.351072:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.351074:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205248.351078:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb000. 02000000:00000001:1.0:1788205248.351079:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205248.351082:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205248.351084:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205248.351087:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205248.351106:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543529600, portal 10 00000100:00000001:1.0:1788205248.351108:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205248.351109:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205248.351114:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351120:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205248.351122:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205248.351124:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205248.351126:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543529600, offset 0 00000400:00000200:1.0:1788205248.351129:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205248.351133:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205248.351136:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205248.351138:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205248.351141:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205248.351142:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205248.351145:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205248.351147:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471486:471486:256:4294967295] 192.168.201.118@tcp LPNI seq info [471345:471345:8:4294967295] 00000400:00000200:1.0:1788205248.351150:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205248.351156:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205248.351160:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.351162:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205248.351166:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205248.351168:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205248.351172:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205248.351174:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205248.351177:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.351179:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205248.351181:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205248.351183:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205248.351185:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205248.351186:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205248.351188:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205248.351190:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351195:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.351197:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351201:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351202:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205248.351203:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205248.351205:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205248.351207:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205248.351208:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321607190 00000100:00000001:1.0:1788205248.351210:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321608455 00000100:00000001:1.0:1788205248.351211:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321607190 00000100:00000001:1.0:1788205248.351213:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321608455 00000100:00000001:1.0:1788205248.351214:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205248.351215:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351216:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205248.351218:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205248/real 1788205248] req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205248.351222:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205248.351223:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205248.351224:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205248.351225:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205248.351226:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205248.351228:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205248.351230:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205248.351234:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351238:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205248.351238:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351239:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205248.351240:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205248.351242:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351245:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205248.351247:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351251:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205248.351252:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351253:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205248.351253:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351254:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205248.351256:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205248.351257:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205248.351259:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205248.351260:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205248.351263:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205248.351267:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529600:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205248.351269:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351271:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.351273:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543529472/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351276:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205248.351277:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205248.351278:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205248.351280:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205248.351281:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205248.351281:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205248.351283:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205248.351284:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413800. 02000000:00000001:1.0:1788205248.351285:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205248.351286:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205248.351287:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205248.351289:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205248.351290:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351291:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.351293:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543529600/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351296:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205248.351297:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205248.351298:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:1.0:1788205248.351299:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205248.351299:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205248.351300:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205248.351301:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205248.351302:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412800. 02000000:00000001:1.0:1788205248.351303:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205248.351303:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205248.351304:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205248.351305:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205248.351306:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351307:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351309:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205248.351310:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351311:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205248.351313:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205248.351313:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205248.351315:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351319:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205248.351320:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.351321:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205248.351322:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205248.351323:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205248.351325:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529728:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205248.351327:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205248.351328:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205248.351329:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.351330:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205248.351332:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb000. 02000000:00000001:1.0:1788205248.351333:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205248.351335:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205248.351336:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205248.351338:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205248.351340:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543529728, portal 4 00000100:00000001:1.0:1788205248.351341:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205248.351341:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205248.351344:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351348:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205248.351349:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205248.351351:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205248.351352:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543529728, offset 0 00000400:00000200:1.0:1788205248.351354:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205248.351356:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205248.351358:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205248.351359:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205248.351361:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205248.351362:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205248.351364:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205248.351366:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471487:471487:256:4294967295] 192.168.201.118@tcp LPNI seq info [471346:471346:8:4294967295] 00000400:00000200:1.0:1788205248.351368:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205248.351372:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205248.351374:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.351376:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205248.351378:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205248.351379:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205248.351382:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205248.351383:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205248.351385:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.351386:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205248.351388:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205248.351390:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205248.351391:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205248.351392:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205248.351393:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205248.351395:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351399:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.351401:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351404:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351405:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205248.351407:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205248.351408:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205248.351410:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205248.351411:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321607199 00000100:00000001:1.0:1788205248.351412:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321608455 00000100:00000001:1.0:1788205248.351413:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321607199 00000100:00000001:1.0:1788205248.351415:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321608455 00000100:00000001:1.0:1788205248.351416:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205248.351417:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351418:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205248.351421:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205248/real 1788205248] req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205248.351424:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205248.351425:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205248.351426:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205248.351427:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205248.351428:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205248.351429:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205248.351430:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205248.351435:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351439:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205248.351439:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351440:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205248.351441:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205248.351443:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351446:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205248.351448:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351451:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205248.351453:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351453:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205248.351454:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351455:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205248.351457:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205248.351458:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205248.351460:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205248.351461:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205248.351463:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205248.351467:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529728:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205248.351469:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205248.351471:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351474:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205248.351475:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.351476:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205248.351477:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205248.351478:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205248.351480:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529856:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205248.351481:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205248.351482:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205248.351483:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205248.351484:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205248.351485:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205248.351500:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205248.351503:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205248.351505:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205248.351507:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205248.351509:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543529856, portal 4 00000100:00000001:1.0:1788205248.351510:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205248.351511:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205248.351515:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351519:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205248.351520:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205248.351522:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205248.351523:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543529856, offset 0 00000400:00000200:1.0:1788205248.351525:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205248.351527:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205248.351529:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205248.351530:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205248.351532:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205248.351533:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205248.351534:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205248.351536:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471488:471488:256:4294967295] 192.168.201.118@tcp LPNI seq info [471347:471347:8:4294967295] 00000400:00000200:1.0:1788205248.351538:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205248.351542:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205248.351545:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.351547:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205248.351549:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205248.351550:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1788205248.351552:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205248.351555:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1788205248.351557:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205248.351558:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205248.351560:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205248.351562:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205248.351563:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205248.351564:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205248.351566:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205248.351568:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351572:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.351574:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205273 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351581:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351582:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205248.351584:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205248.351585:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205248.351587:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205248.351588:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321607222 00000100:00000001:1.0:1788205248.351590:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321608455 00000100:00000001:1.0:1788205248.351591:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321607222 00000100:00000001:1.0:1788205248.351592:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321608455 00000100:00000001:1.0:1788205248.351593:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205248.351595:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351596:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205248.351597:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205248/real 1788205248] req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205248.351601:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205248.351602:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205248.351603:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205248.351604:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205248.351605:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205248.351606:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205248.351607:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205248.351612:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351615:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205248.351616:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351617:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205248.351618:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205248.351620:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351624:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205248.351626:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351629:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205248.351631:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351631:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205248.351632:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351633:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205248.351634:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205248.351635:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205248.351637:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205248.351638:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205248.351640:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205248.351644:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543529856:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205248.351645:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351646:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.351648:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543529728/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351651:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205248.351652:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205248.351653:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:1.0:1788205248.351655:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205248.351656:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205248.351657:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205248.351658:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205248.351659:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410400. 02000000:00000001:1.0:1788205248.351661:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205248.351662:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205248.351662:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205248.351664:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205248.351665:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351666:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205248.351667:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543529856/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205273 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205248.351671:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205248.351671:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205248.351672:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205248.351674:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205248.351674:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205248.351675:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205248.351676:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205248.351676:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411400. 02000000:00000001:1.0:1788205248.351677:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205248.351678:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205248.351679:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205248.351680:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205248.351680:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351681:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205248.351683:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205248.351684:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205248.351685:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205248.351686:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205249.180187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205249.180194:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205249.180194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205249.180194:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205249.180200:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205249.180202:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205249.180203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205249.180205:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205249.180208:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205249.180211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205249.180211:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205249.180215:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205249.180216:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205249.180218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205249.180219:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205249.180221:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205249.180223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205249.180226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205249.180226:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205249.180231:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205249.180232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205249.180233:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205249.180234:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205249.180237:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205249.180241:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205249.180244:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205249.180295:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205249.180297:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205249.180299:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205249.692204:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321608455, at 4321608512 00000100:00000040:2.0:1788205249.692212:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321608455, at 4321608512 00000100:00000010:3.0:1788205249.692217:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647420. 00000100:00000010:2.0:1788205249.692221:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647c60. 00000100:00000001:2.0:1788205250.204221:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205250.204229:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205250.204231:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205250.204237:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205250.204243:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205250.204248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205250.204251:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205250.204254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205250.204255:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205250.204260:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205250.204267:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205250.204272:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205250.204276:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205250.204283:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205250.204286:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205250.204287:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205250.204293:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205250.204294:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205250.204298:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205250.204299:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205250.204301:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205250.204304:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205250.204305:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205250.204309:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205250.204313:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205250.204313:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205250.204316:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205250.204319:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205250.204321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205250.204324:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205250.204328:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205250.204330:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205250.206041:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205250.206049:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205250.206056:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205251.228212:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205251.228223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205251.228223:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205251.228223:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205251.228226:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205251.228230:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205251.228230:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205251.228234:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205251.228235:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205251.228238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205251.228241:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205251.228242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205251.228242:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205251.228247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205251.228249:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205251.228250:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205251.228251:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205251.228254:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205251.228255:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205251.228255:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205251.228260:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205251.228261:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205251.228263:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205251.228264:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205251.228266:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205251.228269:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205251.228531:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205251.228535:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205251.228538:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205252.252167:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205252.252176:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205252.252177:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205252.252180:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205252.252181:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205252.252186:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205252.252188:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205252.252191:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205252.252192:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205252.252196:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205252.252197:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205252.252202:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205252.252202:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205252.252202:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205252.252208:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205252.252209:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205252.252210:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205252.252212:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205252.252213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205252.252214:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205252.252222:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205252.252226:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205252.252230:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205252.252232:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205252.252234:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205252.252237:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205252.252270:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205252.252273:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205252.252275:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205253.276171:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205253.276172:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205253.276180:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205253.276181:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205253.276184:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.276185:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205253.276192:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205253.276192:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205253.276199:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205253.276199:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205253.276203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205253.276204:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205253.276205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205253.276211:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.276213:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205253.276214:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205253.276221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205253.276221:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205253.276223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205253.276226:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205253.276227:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205253.276228:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205253.276235:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205253.276237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205253.276241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205253.276243:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205253.276298:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205253.276302:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205253.276306:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788205253.468142:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205253.468150:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205253.468152:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205253.468155:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205253.468158:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205253.468163:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26939 00000400:00000200:3.0:1788205253.468171:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205253.468222:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205253.468224:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205253.468226:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468228:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205253.468230:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468232:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205253.468235:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205253.468236:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205253.468239:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205253.468242:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205253.468243:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205253.468244:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205253.468246:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468247:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205253.468248:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468250:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205253.468252:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205253.468254:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468255:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205253.468257:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205253.468260:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205253.468261:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468263:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205253.468265:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205253.468274:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39180. 00000020:00000040:3.0:1788205253.468276:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205253.468279:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205253.468280:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468282:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205253.468285:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205253.468294:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205253.468301:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410800. 00000100:00000001:3.0:1788205253.468305:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205253.468311:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205253.468319:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205253.468322:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205253.468325:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000081e50259] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205253.468340:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205253.468342:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205253.468344:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205253.468345:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205253.468347:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205253.468348:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205253.468350:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26939 00000400:00000200:3.0:1788205253.468352:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205253.468354:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205253.468356:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205253.468357:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468359:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205253.468360:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468362:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205253.468363:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205253.468365:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205253.468366:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205253.468368:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205253.468369:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205253.468371:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205253.468373:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468374:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205253.468375:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205253.468377:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205253.468377:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205253.468380:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205253.468382:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205253.468383:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205253.468384:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205253.468387:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788205253.468388:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788205253.468392:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205253.468395:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468396:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000040:1.0:1788205253.468397:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788205253.468410:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205253.468414:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468418:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.468424:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000010:3.0:1788205253.468424:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205253.468428:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205253.468429:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205253.468433:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205253.468434:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205253.468438:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205253.468438:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468441:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205253.468443:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205253.468445:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788205253.468449:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543529984:192.168.201.118@tcp:250:kworker.0 02000000:00000001:3.0:1788205253.468449:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.468457:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1788205253.468457:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205253.468462:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788205253.468463:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.468466:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.468470:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788205253.468471:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1788205253.468476:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb000. 00000100:00000001:3.0:1788205253.468477:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205253.468480:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205253.468483:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003bbc84eb] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205253.468488:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468490:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205253.468493:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1788205253.468498:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205253.468499:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205253.468502:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1788205253.468505:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000100:00080000:3.0:1788205253.468506:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205253.468511:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205253.468514:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26939 00000400:00000010:1.0:1788205253.468516:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000400:00000200:3.0:1788205253.468520:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:1.0:1788205253.468525:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543529984, portal 25 00000400:00000200:3.0:1788205253.468525:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:1.0:1788205253.468530:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788205253.468531:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788205253.468533:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643411840 : -109420066139776 : ffff9c7ba7d39180) 00000100:00000040:3.0:1788205253.468537:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468541:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205253.468543:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205253.468545:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468548:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205253.468552:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205253.468554:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205253.468556:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205253.468558:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000100:00000001:1.0:1788205253.468560:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788205253.468561:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205253.468564:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205253.468568:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00000040:3.0:1788205253.468568:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000010:1.0:1788205253.468573:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205253.468577:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543529984, offset 0 00000100:00000001:3.0:1788205253.468578:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205253.468580:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468581:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205253.468583:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205253.468585:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468588:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205253.468589:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205253.468592:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1788205253.468593:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205253.468595:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468597:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205253.468599:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205253.468601:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468602:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.468606:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000010:3.0:1788205253.468607:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38a80. 00000020:00000040:3.0:1788205253.468610:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205253.468612:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205253.468614:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.468615:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:3.0:1788205253.468616:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205253.468618:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205253.468621:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.468623:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:3.0:1788205253.468624:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411800. 00000100:00000001:3.0:1788205253.468628:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.468630:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00100000:3.0:1788205253.468634:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1788205253.468638:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1788205253.468644:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1788205253.468645:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468646:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205253.468651:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009f49ea7e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205253.468655:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471489:471489:256:4294967295] 192.168.201.118@tcp LPNI seq info [471348:471348:8:4294967295] 00000100:00000001:3.0:1788205253.468662:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.468666:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:3.0:1788205253.468666:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205253.468670:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205253.468671:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205253.468672:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205253.468674:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205253.468678:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26939 00000400:00000200:1.0:1788205253.468683:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000400:00000200:3.0:1788205253.468690:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:1.0:1788205253.468691:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205253.468695:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205253.468697:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000010:1.0:1788205253.468699:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000100:00000040:3.0:1788205253.468700:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468704:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205253.468706:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468707:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205253.468709:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205253.468711:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000800:00000200:1.0:1788205253.468712:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000020:00000001:3.0:1788205253.468713:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205253.468716:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205253.468718:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000800:00000200:1.0:1788205253.468719:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000020:00000001:3.0:1788205253.468720:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205253.468725:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468727:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205253.468732:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:1.1:1788205253.468732:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:3.0:1788205253.468735:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205253.468739:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000800:00000200:1.1:1788205253.468740:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000100:00080000:3.0:1788205253.468741:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468746:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205253.468748:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:1.0:1788205253.468750:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000001:3.0:1788205253.468753:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1788205253.468754:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 02000000:00000001:3.0:1788205253.468757:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468761:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:1.0:1788205253.468762:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00100000:3.0:1788205253.468763:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205253.468769:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.468771:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1788205253.468773:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205253.468780:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a300. 00000020:00000040:3.0:1788205253.468782:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205253.468785:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205253.468786:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468787:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205253.468788:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205253.468790:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.468793:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 02000000:00000010:3.0:1788205253.468795:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000400:00000010:1.0:1788205253.468798:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:3.0:1788205253.468799:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.468805:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1788205253.468806:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205253.468813:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205253.468814:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205253.468817:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205253.468823:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a4465d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.468832:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.468838:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205253.468839:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205253.468844:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205253.468851:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205253.468854:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788205253.468857:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205253.468857:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205253.468861:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205253.468863:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1788205253.468867:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205253.468868:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26939 00000400:00000200:1.0:1788205253.468875:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205253.468876:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205253.468880:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321613573 00000400:00000200:3.0:1788205253.468883:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205253.468886:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000010:1.0:1788205253.468887:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16471e0. 00000100:00000040:3.0:1788205253.468890:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468893:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205253.468895:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205253.468897:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205253.468897:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205253.468901:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205253.468902:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205253.468904:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205253.468905:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000100:00000001:1.0:1788205253.468907:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321613573 00000020:00000001:3.0:1788205253.468907:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205253.468912:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000010:1.0:1788205253.468914:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647540. 00000100:00000040:3.0:1788205253.468915:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205253.468919:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788205253.468920:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000040:3.0:1788205253.468923:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205253.468925:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205253.468927:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:1.0:1788205253.468930:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1788205253.468931:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00000001:1.0:1788205253.468934:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788205253.468937:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00100000:1.0:1788205253.468943:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205253/real 1788205253] req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205253.468943:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205253.468947:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205253.468951:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205253.468956:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788205253.468959:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788205253.468960:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000001:1.0:1788205253.468964:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:3.0:1788205253.468964:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205253.468967:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1788205253.468968:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000100:00000001:3.0:1788205253.468969:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205253.468974:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3a680. 00000400:00000200:1.0:1788205253.468975:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000020:00000040:3.0:1788205253.468977:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000400:00000010:1.0:1788205253.468978:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 02000000:00000001:3.0:1788205253.468980:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788205253.468983:0:256132:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1788205253.468983:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205253.468984:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205253.468987:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205253.468989:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1788205253.468991:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788205253.468992:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788205253.468998:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205253.469005:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1788205253.469010:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205253.469015:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205253.469019:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.469022:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205253.469024:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205253.469027:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000042d14b9d] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469028:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.469031:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205253.469039:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205253.469041:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205253.469049:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26949) 00000100:00000001:1.0:1788205253.469051:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205253.469058:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469065:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205253.469069:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469071:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.469072:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469074:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205253.469076:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205253.469079:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205253.469082:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205253.469085:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205253.469088:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205253.469119:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543529984:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205253.469123:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469125:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.469128:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39180 x1875055543529984/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469134:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205253.469136:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205253.469139:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:1.0:1788205253.469142:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205253.469143:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205253.469145:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205253.469147:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205253.469150:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410800. 02000000:00000001:1.0:1788205253.469153:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205253.469155:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205253.469157:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39180. 00000100:00000001:1.0:1788205253.469159:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205253.469161:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469162:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469169:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205253.469170:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.469172:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205253.469174:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205253.469175:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205253.469179:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469185:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205253.469186:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.469188:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205253.469190:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205253.469192:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205253.469194:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530112:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205253.469197:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205253.469199:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205253.469200:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.469202:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205253.469205:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9bb000. 02000000:00000001:1.0:1788205253.469206:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.469210:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205253.469213:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205253.469216:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205253.469219:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543530112, portal 10 00000100:00000001:1.0:1788205253.469221:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205253.469222:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643419008 : -109420066132608 : ffff9c7ba7d3ad80) 00000100:00000040:1.0:1788205253.469228:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469234:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205253.469236:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205253.469238:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205253.469240:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543530112, offset 0 00000400:00000200:1.0:1788205253.469244:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205253.469247:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205253.469251:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205253.469253:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205253.469256:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205253.469258:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205253.469260:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205253.469263:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471490:471490:256:4294967295] 192.168.201.118@tcp LPNI seq info [471349:471349:8:4294967295] 00000400:00000200:1.0:1788205253.469267:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205253.469273:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205253.469277:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205253.469279:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205253.469283:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205253.469286:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205253.469290:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205253.469293:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205253.469297:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205253.469298:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205253.469301:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205253.469304:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205253.469307:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205253.469309:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205253.469311:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205253.469314:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469321:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.469324:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469329:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.469331:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205253.469333:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205253.469337:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205253.469340:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205253.469342:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321612307 00000100:00000001:1.0:1788205253.469344:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321613573 00000100:00000001:1.0:1788205253.469346:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321612307 00000100:00000001:1.0:1788205253.469348:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321613573 00000100:00000001:1.0:1788205253.469350:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205253.469352:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.469353:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205253.469357:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205253/real 1788205253] req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205253.469364:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205253.469365:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205253.469367:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205253.469369:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205253.469371:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205253.469373:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205253.469375:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205253.469383:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469388:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205253.469389:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469390:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.469392:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205253.469395:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469414:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205253.469418:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469424:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205253.469426:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469428:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.469429:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469431:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205253.469433:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205253.469436:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205253.469439:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205253.469441:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205253.469444:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205253.469451:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530112:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205253.469455:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205253.469457:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469464:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205253.469465:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.469467:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205253.469468:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205253.469470:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205253.469473:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530240:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205253.469476:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205253.469478:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205253.469479:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.469481:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205253.469484:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788205253.469486:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.469489:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205253.469491:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205253.469494:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205253.469496:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543530240, portal 10 00000100:00000001:1.0:1788205253.469498:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205253.469499:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643410048 : -109420066141568 : ffff9c7ba7d38a80) 00000100:00000040:1.0:1788205253.469504:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469515:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205253.469517:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205253.469518:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205253.469520:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543530240, offset 0 00000400:00000200:1.0:1788205253.469524:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205253.469528:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205253.469531:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205253.469533:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205253.469535:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205253.469538:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205253.469540:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205253.469542:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471491:471491:256:4294967295] 192.168.201.118@tcp LPNI seq info [471350:471350:8:4294967295] 00000400:00000200:1.0:1788205253.469546:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205253.469551:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205253.469555:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205253.469558:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205253.469561:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205253.469564:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205253.469568:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205253.469570:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205253.469574:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205253.469575:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205253.469578:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205253.469581:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205253.469584:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205253.469586:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205253.469588:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205253.469591:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469597:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.469600:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469611:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.469612:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205253.469615:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205253.469617:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205253.469620:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205253.469622:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321612308 00000100:00000001:1.0:1788205253.469624:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321613573 00000100:00000001:1.0:1788205253.469627:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321612308 00000100:00000001:1.0:1788205253.469628:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321613573 00000100:00000001:1.0:1788205253.469630:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205253.469632:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.469634:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205253.469637:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205253/real 1788205253] req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205253.469644:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205253.469645:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205253.469647:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205253.469649:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205253.469650:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205253.469652:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205253.469655:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205253.469662:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469667:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205253.469669:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469670:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.469671:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205253.469675:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469681:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205253.469684:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469690:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205253.469692:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469693:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.469694:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469696:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205253.469698:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205253.469700:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205253.469702:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205253.469704:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205253.469708:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205253.469715:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530240:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205253.469718:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205253.469721:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469727:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205253.469729:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.469730:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205253.469732:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205253.469734:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205253.469737:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530368:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205253.469739:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205253.469741:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205253.469743:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.469745:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205253.469747:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205253.469749:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.469752:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205253.469754:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205253.469757:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205253.469759:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543530368, portal 4 00000100:00000001:1.0:1788205253.469761:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205253.469763:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643416320 : -109420066135296 : ffff9c7ba7d3a300) 00000100:00000040:1.0:1788205253.469767:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469772:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205253.469774:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205253.469776:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205253.469778:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543530368, offset 0 00000400:00000200:1.0:1788205253.469781:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205253.469785:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205253.469788:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205253.469790:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205253.469792:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205253.469794:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205253.469796:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205253.469799:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471492:471492:256:4294967295] 192.168.201.118@tcp LPNI seq info [471351:471351:8:4294967295] 00000400:00000200:1.0:1788205253.469802:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205253.469808:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205253.469811:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205253.469814:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205253.469817:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205253.469819:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205253.469823:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205253.469825:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205253.469828:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205253.469830:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205253.469833:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205253.469836:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205253.469838:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205253.469839:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205253.469842:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205253.469844:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469851:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.469854:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469860:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.469861:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205253.469863:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205253.469866:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205253.469868:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205253.469870:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321612317 00000100:00000001:1.0:1788205253.469873:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321613573 00000100:00000001:1.0:1788205253.469874:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321612317 00000100:00000001:1.0:1788205253.469876:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321613573 00000100:00000001:1.0:1788205253.469878:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205253.469880:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.469882:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205253.469885:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205253/real 1788205253] req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205253.469892:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205253.469893:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205253.469895:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205253.469897:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205253.469898:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205253.469900:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205253.469903:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205253.469911:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469916:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205253.469918:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469919:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.469921:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205253.469924:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469930:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205253.469933:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469939:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205253.469941:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469943:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.469944:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.469946:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205253.469947:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205253.469949:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205253.469952:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205253.469954:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205253.469957:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205253.469964:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530368:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205253.469967:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205253.469971:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.469977:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205253.469978:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.469980:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205253.469981:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205253.469983:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205253.469986:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530496:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205253.469989:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205253.469990:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205253.469991:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205253.469993:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205253.469996:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205253.469997:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205253.470000:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205253.470002:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205253.470005:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205253.470007:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543530496, portal 4 00000100:00000001:1.0:1788205253.470009:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205253.470010:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643417216 : -109420066134400 : ffff9c7ba7d3a680) 00000100:00000040:1.0:1788205253.470015:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470020:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205253.470022:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205253.470024:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205253.470026:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543530496, offset 0 00000400:00000200:1.0:1788205253.470030:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205253.470033:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205253.470036:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205253.470038:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205253.470041:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205253.470043:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205253.470045:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205253.470047:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471493:471493:256:4294967295] 192.168.201.118@tcp LPNI seq info [471352:471352:8:4294967295] 00000400:00000200:1.0:1788205253.470051:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205253.470057:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205253.470062:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205253.470064:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205253.470068:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205253.470070:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1788205253.470074:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205253.470076:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1788205253.470080:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205253.470082:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205253.470084:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205253.470087:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205253.470103:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205253.470106:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205253.470108:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205253.470111:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470118:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.470121:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205278 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470126:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.470128:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205253.470130:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205253.470133:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205253.470136:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205253.470138:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321612341 00000100:00000001:1.0:1788205253.470140:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321613573 00000100:00000001:1.0:1788205253.470142:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321612341 00000100:00000001:1.0:1788205253.470144:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321613573 00000100:00000001:1.0:1788205253.470146:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205253.470148:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.470150:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205253.470153:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205253/real 1788205253] req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205253.470160:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205253.470161:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205253.470163:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205253.470165:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205253.470166:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205253.470168:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205253.470171:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205253.470179:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470184:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205253.470186:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470188:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.470189:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205253.470192:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470198:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205253.470201:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470208:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205253.470210:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470211:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205253.470212:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470213:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205253.470215:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205253.470217:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205253.470220:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205253.470222:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205253.470225:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205253.470233:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530496:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205253.470236:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470238:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.470240:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3ad80 x1875055543530112/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470246:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205253.470247:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205253.470249:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:1.0:1788205253.470251:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205253.470252:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205253.470254:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205253.470256:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205253.470257:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410c00. 02000000:00000001:1.0:1788205253.470259:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205253.470260:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205253.470261:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3ad80. 00000100:00000001:1.0:1788205253.470264:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205253.470265:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470267:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.470269:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38a80 x1875055543530240/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470275:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205253.470276:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205253.470277:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:1.0:1788205253.470279:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205253.470280:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205253.470282:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205253.470283:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205253.470285:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411800. 02000000:00000001:1.0:1788205253.470287:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205253.470288:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205253.470290:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38a80. 00000100:00000001:1.0:1788205253.470293:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205253.470294:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470295:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.470298:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a300 x1875055543530368/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470304:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205253.470305:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205253.470306:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:1.0:1788205253.470308:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205253.470309:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205253.470310:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205253.470311:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205253.470313:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205253.470315:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205253.470316:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205253.470317:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a300. 00000100:00000001:1.0:1788205253.470319:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205253.470320:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470321:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205253.470323:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3a680 x1875055543530496/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205278 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205253.470328:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205253.470329:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205253.470331:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:1.0:1788205253.470332:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205253.470333:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205253.470334:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205253.470336:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205253.470337:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205253.470339:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205253.470340:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205253.470341:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3a680. 00000100:00000001:1.0:1788205253.470343:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205253.470345:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470346:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205253.470349:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205253.470350:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205253.470352:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205253.470354:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205254.300224:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205254.300235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205254.300237:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205254.300241:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205254.300243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205254.300246:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205254.300254:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205254.300259:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205254.300260:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205254.300265:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205254.300267:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205254.300269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205254.300272:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205254.300275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205254.300277:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205254.300279:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205254.300282:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205254.300285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205254.300288:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205254.300289:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205254.300446:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205254.300450:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205254.300454:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788205254.301370:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205254.301372:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205254.301374:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205254.301376:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205254.301379:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205254.301381:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205254.812196:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321613573, at 4321613632 00000100:00000040:2.0:1788205254.812216:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321613573, at 4321613632 00000100:00000010:3.0:1788205254.813827:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647540. 00000100:00000010:2.0:1788205254.813831:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16471e0. 00000100:00000001:2.0:1788205255.324198:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205255.324198:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205255.324203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205255.324206:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205255.324207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205255.324208:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205255.324210:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205255.324211:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205255.324217:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205255.324217:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205255.324218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205255.324222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205255.324223:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205255.324225:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205255.324226:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205255.324227:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205255.324228:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205255.324232:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205255.324233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205255.324234:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205255.324237:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205255.324237:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205255.324239:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205255.324243:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205255.324244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205255.324244:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324246:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324252:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324255:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205255.324257:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205255.324259:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205255.324261:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324263:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205255.324264:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324266:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324267:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205255.324272:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205255.324273:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205255.324274:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324276:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205255.324277:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324278:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205255.324280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205255.324399:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205255.324402:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205255.324404:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205256.348200:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205256.348200:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205256.348206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205256.348207:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205256.348207:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205256.348210:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.348212:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205256.348213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205256.348217:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205256.348218:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205256.348221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205256.348224:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205256.348225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.348225:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205256.348229:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205256.348230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205256.348232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205256.348237:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.348237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205256.348237:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205256.348243:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205256.348243:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205256.348244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205256.348249:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205256.348250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205256.348250:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205256.348329:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205256.348332:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205256.348333:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205257.372206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205257.372208:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205257.372213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205257.372215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205257.372219:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205257.372219:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205257.372223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205257.372226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205257.372228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205257.372231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205257.372233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205257.372237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205257.372238:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205257.372240:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205257.372242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205257.372247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205257.372250:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205257.373012:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205257.373017:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205257.373018:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205257.373020:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205257.373023:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205257.373025:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205257.373027:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205257.373029:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205257.373031:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205257.374439:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205257.374445:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205257.374449:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205258.396213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205258.396213:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205258.396218:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205258.396223:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205258.396223:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205258.396224:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205258.396229:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.396229:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205258.396232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205258.396236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205258.396236:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205258.396236:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205258.396243:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205258.396244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205258.396244:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205258.396248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205258.396249:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.396250:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205258.396252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205258.396255:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205258.396256:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.396260:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205258.396265:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205258.396266:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205258.396269:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205258.396273:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205258.396417:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205258.396421:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205258.396425:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205258.588802:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205258.588810:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205258.588813:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205258.588816:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205258.588821:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205258.588828:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26944 00000400:00000200:1.0:1788205258.588838:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205258.588844:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205258.588847:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205258.588850:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.588853:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205258.588855:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.588858:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205258.588862:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205258.588864:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205258.588867:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205258.588870:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205258.588872:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205258.588874:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205258.588876:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.588877:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205258.588878:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.588880:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205258.588882:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205258.588884:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205258.588886:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205258.588888:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205258.588890:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205258.588892:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.588893:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205258.588895:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205258.588905:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000020:00000040:1.0:1788205258.588909:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205258.588912:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205258.588914:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.588917:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205258.588919:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205258.588924:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205258.588932:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8000. 00000100:00000001:1.0:1788205258.588939:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205258.588947:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205258.588957:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205258.588961:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205258.588966:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205258.588991:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205258.588995:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205258.588998:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205258.588999:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205258.589001:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205258.589004:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205258.589006:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26944 00000400:00000200:1.0:1788205258.589011:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205258.589015:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205258.589018:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205258.589019:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1788205258.589022:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205258.589025:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205258.589026:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205258.589029:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205258.589030:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589034:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205258.589038:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000100:00000040:3.0:1788205258.589038:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205258.589041:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205258.589045:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205258.589049:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000100:00000001:3.0:1788205258.589053:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1788205258.589054:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205258.589057:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205258.589058:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 02000000:00000001:3.0:1788205258.589063:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1788205258.589065:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 02000000:00000001:3.0:1788205258.589067:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205258.589069:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788205258.589072:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205258.589073:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589077:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00100000:3.0:1788205258.589079:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543530624:192.168.201.118@tcp:250:kworker.0 00000020:00000040:1.0:1788205258.589081:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205258.589087:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205258.589088:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205258.589114:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205258.589116:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589119:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.589120:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788205258.589123:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:3.0:1788205258.589127:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788205258.589129:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205258.589132:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205258.589133:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589137:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205258.589142:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000400:00000200:3.0:1788205258.589143:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000008:00000001:1.0:1788205258.589145:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589147:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205258.589149:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000100:00000010:1.0:1788205258.589151:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000020:00000040:1.0:1788205258.589155:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205258.589157:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205258.589157:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589159:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:3.0:1788205258.589159:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 02000000:00000001:1.0:1788205258.589162:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205258.589165:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205258.589169:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8800. 00000100:00000200:3.0:1788205258.589169:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543530624, portal 25 00000100:00000001:1.0:1788205258.589174:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.589174:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205258.589179:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00100000:1.0:1788205258.589181:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:3.0:1788205258.589189:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205258.589190:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205258.589193:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205258.589199:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.589206:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788205258.589210:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205258.589214:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1788205258.589214:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000100:00080000:1.0:1788205258.589217:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205258.589219:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205258.589220:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205258.589222:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205258.589225:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26944 00000400:00000200:1.0:1788205258.589231:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205258.589232:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000400:00000200:1.0:1788205258.589236:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000200:3.0:1788205258.589237:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543530624, offset 0 00000100:00080000:1.0:1788205258.589240:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205258.589245:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589249:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205258.589249:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000040:1.0:1788205258.589255:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589257:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205258.589261:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000200:3.0:1788205258.589262:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:1.0:1788205258.589263:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205258.589267:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205258.589269:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000400:00000200:3.0:1788205258.589272:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000020:00000001:1.0:1788205258.589273:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205258.589278:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205258.589279:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1788205258.589284:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589287:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205258.589289:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589291:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205258.589293:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000400:00000200:3.0:1788205258.589297:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205258.589303:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205258.589306:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00080000:1.0:1788205258.589308:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000200:3.0:1788205258.589327:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471494:471494:256:4294967295] 192.168.201.118@tcp LPNI seq info [471353:471353:8:4294967295] 00000400:00000200:3.0:1788205258.589332:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205258.589341:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205258.589344:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:1.0:1788205258.589345:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205258.589350:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205258.589355:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205258.589356:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589359:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205258.589363:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205258.589365:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589367:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205258.589372:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205258.589374:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205258.589376:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205258.589377:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589379:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205258.589380:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205258.589382:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205258.589386:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8400. 00000100:00000001:1.0:1788205258.589390:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205258.589394:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205258.589401:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205258.589403:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205258.589407:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:3.0:1788205258.589407:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000100:00000001:1.0:1788205258.589417:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205258.589422:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:3.0:1788205258.589422:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00080000:1.0:1788205258.589429:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:3.0:1788205258.589430:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00000001:1.0:1788205258.589431:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205258.589433:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205258.589436:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205258.589440:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26944 00000400:00000200:1.0:1788205258.589447:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000800:00000200:3.1:1788205258.589450:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000400:00000200:1.0:1788205258.589454:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:3.1:1788205258.589456:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000100:00080000:1.0:1788205258.589458:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205258.589461:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589463:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205258.589467:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589473:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000100:3.0:1788205258.589473:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000020:00000001:1.0:1788205258.589478:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000800:00000010:3.0:1788205258.589478:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000020:00000040:1.0:1788205258.589482:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205258.589485:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205258.589487:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000400:00000200:3.0:1788205258.589487:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:1.0:1788205258.589489:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205258.589493:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205258.589497:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205258.589499:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1788205258.589500:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205258.589504:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589506:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205258.589508:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000020:00000040:1.0:1788205258.589509:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205258.589513:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000400:00000010:3.0:1788205258.589514:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:1.0:1788205258.589515:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205258.589519:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205258.589521:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205258.589522:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1788205258.589523:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589526:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205258.589528:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000200:3.0:1788205258.589530:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1788205258.589531:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589534:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205258.589539:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205258.589544:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205258.589548:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205258.589548:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788205258.589552:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589555:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:3.0:1788205258.589556:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205258.589558:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205258.589561:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205258.589568:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb000. 00000100:00000001:3.0:1788205258.589569:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589574:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.589574:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1788205258.589583:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205258.589583:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205258.589591:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205258.589596:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321618694 00000100:00000001:1.0:1788205258.589597:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205258.589601:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205258.589603:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc000. 00000100:00000040:1.0:1788205258.589606:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.589612:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205258.589619:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.589620:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321618694 00000100:00080000:1.0:1788205258.589624:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:3.0:1788205258.589626:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc8a0. 00000100:00080000:1.0:1788205258.589629:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205258.589632:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205258.589633:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205258.589636:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205258.589636:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205258.589642:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589643:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205258.589646:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26944 00000100:00000001:3.0:1788205258.589648:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000400:00000200:1.0:1788205258.589652:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00100000:3.0:1788205258.589655:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205258/real 1788205258] req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205258.589657:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205258.589661:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205258.589665:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589669:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205258.589673:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589677:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000001:3.0:1788205258.589679:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1788205258.589680:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000400:00000001:3.0:1788205258.589682:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:1.0:1788205258.589684:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000400:00000010:3.0:1788205258.589686:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000020:00000001:1.0:1788205258.589687:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000400:00000200:3.0:1788205258.589690:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000020:00000040:1.0:1788205258.589692:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000400:00000010:3.0:1788205258.589695:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000020:00000001:1.0:1788205258.589696:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205258.589700:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1788205258.589701:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000200:3.0:1788205258.589707:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205258.589708:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589713:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205258.589717:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589722:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000200:3.0:1788205258.589726:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205258.589727:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205258.589732:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205258.589736:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205258.589740:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205258.589743:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205258.589744:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205258.589747:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788205258.589748:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589752:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205258.589752:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:1.0:1788205258.589754:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1788205258.589756:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1788205258.589757:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589761:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205258.589765:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000100:00100000:3.0:1788205258.589766:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1788205258.589769:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205258.589773:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205258.589775:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205258.589778:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205258.589781:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205258.589783:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.589784:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1788205258.589786:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9400. 00000100:00000001:1.0:1788205258.589790:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205258.589792:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205258.589796:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205258.589803:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205258.589805:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205258.589807:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205258.589819:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.589825:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1788205258.589826:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26955) 00000100:00000001:3.0:1788205258.589829:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.589833:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.589834:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.589837:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205258.589840:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205258.589843:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205258.589848:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205258.589851:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205258.589855:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205258.589864:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543530624:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205258.589878:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.589880:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.589884:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543530624/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.589890:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205258.589893:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205258.589896:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205258.589898:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205258.589900:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205258.589902:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205258.589904:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205258.589907:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:3.0:1788205258.589911:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205258.589913:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205258.589914:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205258.589921:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205258.589922:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.589924:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.589932:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205258.589934:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.589935:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205258.589937:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205258.589939:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205258.589942:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.589949:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205258.589950:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.589952:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205258.589954:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205258.589956:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205258.589958:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530752:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205258.589962:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205258.589964:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205258.589965:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.589967:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205258.589969:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205258.589971:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205258.589975:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205258.589978:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205258.589981:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205258.589985:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543530752, portal 10 00000100:00000001:3.0:1788205258.589986:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205258.589988:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00000040:3.0:1788205258.589995:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590002:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205258.590004:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205258.590006:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205258.590008:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543530752, offset 0 00000400:00000200:3.0:1788205258.590012:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205258.590016:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205258.590020:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205258.590023:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205258.590026:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205258.590028:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205258.590030:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205258.590033:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471495:471495:256:4294967295] 192.168.201.118@tcp LPNI seq info [471354:471354:8:4294967295] 00000400:00000200:3.0:1788205258.590038:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205258.590044:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205258.590048:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205258.590051:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205258.590055:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205258.590058:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788205258.590062:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205258.590065:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1788205258.590068:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205258.590070:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205258.590073:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205258.590076:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205258.590079:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205258.590081:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205258.590083:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205258.590087:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590114:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.590117:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590123:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.590125:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205258.590128:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205258.590132:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205258.590135:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205258.590137:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321617428 00000100:00000001:3.0:1788205258.590139:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321618694 00000100:00000001:3.0:1788205258.590141:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321617428 00000100:00000001:3.0:1788205258.590143:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321618694 00000100:00000001:3.0:1788205258.590145:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205258.590146:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.590148:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205258.590152:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205258/real 1788205258] req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205258.590159:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205258.590161:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205258.590162:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205258.590165:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205258.590166:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205258.590168:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205258.590171:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205258.590178:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590184:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205258.590185:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590187:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.590188:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205258.590192:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590198:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205258.590202:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590207:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205258.590210:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590212:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.590214:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590215:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205258.590217:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205258.590220:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205258.590223:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205258.590225:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205258.590229:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205258.590236:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530752:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205258.590239:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205258.590243:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590249:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205258.590250:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.590252:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205258.590253:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205258.590255:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205258.590258:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530880:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205258.590261:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205258.590263:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205258.590264:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.590265:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205258.590269:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205258.590271:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205258.590274:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205258.590276:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205258.590279:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205258.590282:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543530880, portal 10 00000100:00000001:3.0:1788205258.590284:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205258.590285:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205258.590290:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590296:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205258.590298:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205258.590300:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205258.590302:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543530880, offset 0 00000400:00000200:3.0:1788205258.590306:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205258.590322:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205258.590325:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205258.590328:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205258.590330:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205258.590332:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205258.590335:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205258.590337:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471496:471496:256:4294967295] 192.168.201.118@tcp LPNI seq info [471355:471355:8:4294967295] 00000400:00000200:3.0:1788205258.590341:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205258.590347:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205258.590352:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205258.590355:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205258.590359:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205258.590361:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788205258.590365:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205258.590368:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1788205258.590372:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205258.590373:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205258.590376:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205258.590380:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205258.590383:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205258.590385:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205258.590387:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205258.590390:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590402:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.590406:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590412:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.590413:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205258.590416:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205258.590419:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205258.590422:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205258.590423:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321617429 00000100:00000001:3.0:1788205258.590425:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321618694 00000100:00000001:3.0:1788205258.590427:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321617429 00000100:00000001:3.0:1788205258.590429:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321618694 00000100:00000001:3.0:1788205258.590431:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205258.590432:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.590434:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205258.590438:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205258/real 1788205258] req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205258.590445:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205258.590446:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205258.590447:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205258.590449:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205258.590451:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205258.590453:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205258.590456:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205258.590463:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590470:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205258.590471:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590473:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.590474:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205258.590478:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590484:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205258.590487:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590494:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205258.590496:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590497:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.590499:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590500:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205258.590503:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205258.590505:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205258.590508:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205258.590510:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205258.590513:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205258.590520:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543530880:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205258.590523:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205258.590526:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590532:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205258.590533:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.590535:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205258.590537:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205258.590538:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205258.590542:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531008:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205258.590544:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205258.590546:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205258.590548:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.590550:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205258.590553:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205258.590555:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205258.590558:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205258.590560:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205258.590563:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205258.590566:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543531008, portal 4 00000100:00000001:3.0:1788205258.590568:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205258.590569:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205258.590574:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590580:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205258.590582:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205258.590584:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205258.590586:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543531008, offset 0 00000400:00000200:3.0:1788205258.590590:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205258.590594:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205258.590596:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205258.590599:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205258.590601:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205258.590603:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205258.590605:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205258.590608:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471497:471497:256:4294967295] 192.168.201.118@tcp LPNI seq info [471356:471356:8:4294967295] 00000400:00000200:3.0:1788205258.590611:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205258.590616:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205258.590620:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205258.590622:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205258.590626:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205258.590628:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788205258.590632:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205258.590634:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1788205258.590637:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205258.590639:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205258.590642:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205258.590645:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205258.590648:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205258.590649:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205258.590652:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205258.590655:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590661:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.590664:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590670:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.590671:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205258.590686:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205258.590689:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205258.590692:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205258.590694:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321617438 00000100:00000001:3.0:1788205258.590697:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321618694 00000100:00000001:3.0:1788205258.590699:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321617438 00000100:00000001:3.0:1788205258.590701:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321618694 00000100:00000001:3.0:1788205258.590703:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205258.590704:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.590706:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205258.590710:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205258/real 1788205258] req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205258.590717:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205258.590718:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205258.590719:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205258.590721:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205258.590723:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205258.590725:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205258.590728:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205258.590735:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590740:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205258.590742:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590743:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.590744:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205258.590748:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590754:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205258.590758:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590763:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205258.590766:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590767:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.590768:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590770:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205258.590772:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205258.590774:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205258.590777:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205258.590779:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205258.590783:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205258.590794:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531008:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205258.590797:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205258.590800:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590805:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205258.590807:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.590809:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205258.590810:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205258.590812:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205258.590815:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531136:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205258.590818:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205258.590820:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205258.590821:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205258.590823:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205258.590826:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788205258.590828:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205258.590830:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205258.590833:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205258.590836:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205258.590838:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543531136, portal 4 00000100:00000001:3.0:1788205258.590840:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205258.590841:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205258.590846:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590852:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205258.590854:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205258.590856:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205258.590858:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543531136, offset 0 00000400:00000200:3.0:1788205258.590861:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205258.590865:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205258.590868:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205258.590871:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205258.590873:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205258.590875:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205258.590877:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205258.590879:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471498:471498:256:4294967295] 192.168.201.118@tcp LPNI seq info [471357:471357:8:4294967295] 00000400:00000200:3.0:1788205258.590883:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205258.590888:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205258.590891:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205258.590894:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205258.590897:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205258.590899:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000200:3.1:1788205258.590903:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205258.590905:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 42 secs later) 00000800:00000100:3.0:1788205258.590908:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205258.590910:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205258.590912:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205258.590915:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205258.590917:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205258.590919:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205258.590921:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205258.590924:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590930:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.590932:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205283 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590938:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.590939:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205258.590941:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205258.590944:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205258.590946:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205258.590949:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321617461 00000100:00000001:3.0:1788205258.590951:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321618694 00000100:00000001:3.0:1788205258.590953:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321617461 00000100:00000001:3.0:1788205258.590954:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321618694 00000100:00000001:3.0:1788205258.590956:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205258.590958:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.590959:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205258.590962:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205258/real 1788205258] req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205258.590968:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205258.590969:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205258.590970:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205258.590972:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205258.590974:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205258.590976:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205258.590979:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205258.590985:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.590990:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205258.590991:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.590992:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.590994:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205258.590997:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.591002:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205258.591005:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.591010:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205258.591012:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.591013:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205258.591015:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.591016:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205258.591018:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205258.591020:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205258.591022:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205258.591025:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205258.591028:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205258.591034:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531136:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205258.591037:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.591038:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.591041:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543530752/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.591047:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205258.591048:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205258.591050:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205258.591052:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205258.591053:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205258.591055:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205258.591056:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205258.591058:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8800. 02000000:00000001:3.0:1788205258.591060:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205258.591061:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205258.591063:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205258.591067:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205258.591068:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.591070:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.591073:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543530880/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.591082:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205258.591083:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205258.591085:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205258.591086:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205258.591087:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205258.591088:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205258.591109:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205258.591111:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8400. 02000000:00000001:3.0:1788205258.591113:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205258.591114:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205258.591115:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205258.591118:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205258.591119:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.591120:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.591123:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543531008/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.591128:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205258.591129:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205258.591131:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205258.591133:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205258.591134:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205258.591135:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205258.591137:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205258.591138:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:3.0:1788205258.591139:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205258.591140:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205258.591141:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205258.591145:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205258.591145:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.591147:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205258.591149:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543531136/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205283 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205258.591154:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205258.591155:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205258.591157:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788205258.591159:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205258.591160:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205258.591161:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205258.591163:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205258.591164:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:3.0:1788205258.591165:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205258.591166:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205258.591167:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205258.591169:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205258.591170:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.591172:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205258.591175:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205258.591176:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205258.591178:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205258.591179:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205259.420293:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205259.420327:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205259.420334:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205259.420340:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205259.420343:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205259.420343:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205259.420347:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205259.420349:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205259.420351:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205259.420353:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205259.420355:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205259.420358:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205259.420359:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205259.420363:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205259.420367:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205259.420370:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205259.420373:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205259.420590:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205259.420595:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205259.420597:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205259.421625:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205259.421630:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205259.421632:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205259.421648:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205259.421651:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205259.421654:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205259.421656:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205259.421812:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205259.421815:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205259.932200:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321618694, at 4321618752 00000100:00000040:3.0:1788205259.932200:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321618694, at 4321618752 00000100:00000010:3.0:1788205259.932215:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc8a0. 00000100:00000010:2.0:1788205259.932217:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc000. 00000100:00000001:2.0:1788205260.444187:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205260.444187:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205260.444191:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205260.444194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205260.444195:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205260.444197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444198:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205260.444199:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444205:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788205260.444206:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205260.444207:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205260.444212:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205260.444212:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205260.444217:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205260.444220:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205260.444221:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205260.444223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205260.444224:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205260.444228:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205260.444231:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205260.444234:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444253:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444258:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205260.444267:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444269:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444271:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205260.444276:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444287:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205260.444288:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205260.444291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205260.444292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205260.444340:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205260.444344:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205260.444347:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205261.468203:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205261.468213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205261.468214:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205261.468220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205261.468221:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205261.468224:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205261.468225:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205261.468226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205261.468231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205261.468231:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205261.468236:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205261.468239:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205261.468242:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205261.468243:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205261.468244:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205261.468246:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205261.468248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205261.468250:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205261.468253:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205261.468254:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205261.468257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205261.468259:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205261.468259:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205261.468282:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205261.468294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205261.468296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205261.468388:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205261.468392:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205261.468395:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205262.492208:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205262.492216:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205262.492217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205262.492223:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205262.492227:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205262.492230:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205262.492233:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205262.492236:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205262.492239:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205262.492490:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205262.492496:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205262.492498:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205262.492501:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205262.492504:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205262.492506:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205262.492508:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205262.492511:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205262.492512:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205262.493858:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205262.493864:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205262.493871:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205262.493872:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205262.493875:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205262.493876:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205262.493879:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205262.493881:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205262.494043:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205262.494046:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205262.494048:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205263.516262:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205263.516262:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205263.516272:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205263.516272:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205263.516275:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205263.516276:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205263.516284:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205263.516285:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205263.516292:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205263.516292:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205263.516298:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.516299:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205263.516301:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205263.516304:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205263.516305:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205263.516308:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205263.516313:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.516314:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205263.516319:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205263.516323:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205263.516323:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205263.516326:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205263.516332:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205263.516333:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205263.516337:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205263.516339:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205263.516636:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205263.516640:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205263.516643:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205263.708213:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205263.708235:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205263.708239:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205263.708249:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205263.708254:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205263.708261:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26950 00000400:00000200:1.0:1788205263.708271:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205263.708297:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205263.708300:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205263.708303:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708311:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708314:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708319:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205263.708324:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205263.708325:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205263.708328:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205263.708332:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205263.708335:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205263.708338:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205263.708341:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708344:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708345:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708347:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205263.708351:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205263.708354:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708357:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205263.708361:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205263.708365:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205263.708367:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708377:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205263.708380:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205263.708404:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788205263.708408:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205263.708412:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205263.708414:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708417:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205263.708420:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205263.708424:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205263.708433:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba800. 00000100:00000001:1.0:1788205263.708439:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205263.708456:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205263.708467:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205263.708472:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205263.708476:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205263.708497:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205263.708499:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205263.708502:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205263.708503:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205263.708504:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205263.708507:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205263.708509:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26950 00000400:00000200:1.0:1788205263.708512:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205263.708513:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205263.708515:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205263.708525:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708534:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708537:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708539:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205263.708541:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205263.708543:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205263.708545:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205263.708547:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205263.708549:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205263.708551:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205263.708554:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708556:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708557:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708559:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205263.708560:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205263.708563:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708565:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205263.708567:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205263.708573:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205263.708575:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708577:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205263.708592:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205263.708596:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708598:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205263.708602:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205263.708604:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205263.708606:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205263.708607:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708609:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205263.708610:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205263.708611:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205263.708615:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb400. 00000100:00000001:1.0:1788205263.708618:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205263.708623:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205263.708630:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205263.708632:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205263.708636:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205263.708648:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205263.708651:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205263.708653:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205263.708655:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205263.708663:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205263.708666:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205263.708669:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26950 00000400:00000200:1.0:1788205263.708673:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205263.708675:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205263.708677:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205263.708679:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708681:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708682:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708684:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205263.708686:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205263.708686:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205263.708688:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205263.708689:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205263.708691:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205263.708693:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205263.708695:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708697:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708698:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708700:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205263.708702:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205263.708705:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708707:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205263.708709:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205263.708716:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205263.708718:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708720:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205263.708722:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205263.708724:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708725:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205263.708730:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205263.708732:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205263.708734:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205263.708735:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708737:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205263.708738:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205263.708739:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205263.708743:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8c00. 00000100:00000001:1.0:1788205263.708746:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205263.708750:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205263.708757:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205263.708759:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205263.708762:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205263.708767:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205263.708769:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205263.708771:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205263.708772:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205263.708773:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205263.708775:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205263.708777:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26950 00000400:00000200:1.0:1788205263.708780:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205263.708781:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205263.708783:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205263.708785:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708786:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708787:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708789:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205263.708790:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205263.708791:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205263.708792:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205263.708793:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205263.708794:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205263.708795:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205263.708797:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708798:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708799:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708800:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205263.708801:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205263.708803:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708804:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205263.708805:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205263.708807:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205263.708808:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708809:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205263.708810:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205263.708812:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708813:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205263.708815:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205263.708816:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205263.708818:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205263.708819:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708820:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205263.708821:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205263.708822:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205263.708824:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9400. 00000100:00000001:1.0:1788205263.708827:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205263.708829:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205263.708835:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205263.708837:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205263.708840:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205263.708847:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205263.708849:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205263.708852:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205263.708853:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205263.708854:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205263.708856:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205263.708859:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26950 00000400:00000200:1.0:1788205263.708862:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205263.708865:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205263.708867:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205263.708869:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708871:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708873:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708874:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205263.708876:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205263.708877:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205263.708879:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205263.708880:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205263.708882:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:1.0:1788205263.708884:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:1.0:1788205263.708887:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708888:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205263.708891:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708892:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205263.708894:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205263.708897:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205263.708899:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205263.708900:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205263.708902:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205263.708904:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708906:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205263.708908:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205263.708910:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708911:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205263.708914:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205263.708916:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205263.708919:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205263.708920:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205263.708921:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205263.708923:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205263.708924:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205263.708928:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb000. 00000100:00000001:1.0:1788205263.708931:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205263.708935:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205263.708942:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205263.708943:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205263.708947:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205263.708955:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205263.708959:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26960) 00000100:00000001:3.0:1788205263.709562:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205263.709567:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205263.709570:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205263.709578:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.709586:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205263.709588:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.709592:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205263.709594:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205263.709597:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205263.709600:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543531264:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205263.709604:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205263.709608:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205263.709610:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.709612:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205263.709617:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788205263.709619:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205263.709624:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205263.709628:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205263.709635:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205263.709641:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543531264, portal 25 00000100:00000001:3.0:1788205263.709643:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205263.709645:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000040:3.0:1788205263.709649:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.709656:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205263.709659:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205263.709661:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205263.709664:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543531264, offset 0 00000400:00000200:3.0:1788205263.709670:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205263.709677:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.709683:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205263.709688:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205263.709691:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205263.709695:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205263.709697:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205263.709701:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471499:471499:256:4294967295] 192.168.201.118@tcp LPNI seq info [471358:471358:8:4294967295] 00000400:00000200:3.0:1788205263.709706:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.709715:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205263.709720:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.709725:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205263.709731:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205263.709735:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205263.709741:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205263.709744:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205263.709748:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.709751:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205263.709758:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205263.709762:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205263.709768:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205263.709770:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205263.709775:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205263.709779:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.709788:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.709791:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.709798:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.709800:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205263.709806:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205263.709810:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205263.709813:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321623814 00000100:00000010:3.0:1788205263.709819:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc8a0. 00000100:00000001:3.0:1788205263.709825:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205263.709831:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321623814 00000100:00000010:3.0:1788205263.709834:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc780. 00000100:00000001:3.0:1788205263.709837:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205263.709840:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205263.709843:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.709845:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205263.709849:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205263/real 1788205263] req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205263.709858:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205263.709859:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205263.709861:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205263.709864:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205263.709866:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205263.709868:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205263.709872:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205263.709880:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.709886:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205263.709888:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.709890:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.709892:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205263.709896:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.709902:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205263.709906:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.709913:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205263.709916:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.709918:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.709919:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.709921:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205263.709923:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205263.709926:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205263.709930:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205263.709932:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205263.709936:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205263.709943:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543531264:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205263.709949:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205263.709952:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.709958:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205263.709959:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.709961:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205263.709963:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205263.709965:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205263.709967:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531392:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205263.709970:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205263.709972:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205263.709973:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.709975:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205263.709977:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205263.709980:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205263.709983:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205263.709985:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205263.709988:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205263.709990:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543531392, portal 10 00000100:00000001:3.0:1788205263.709992:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205263.709993:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788205263.709999:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710005:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205263.710007:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205263.710009:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205263.710011:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543531392, offset 0 00000400:00000200:3.0:1788205263.710015:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205263.710018:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.710022:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205263.710024:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205263.710027:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205263.710029:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205263.710031:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205263.710034:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471500:471500:256:4294967295] 192.168.201.118@tcp LPNI seq info [471359:471359:8:4294967295] 00000400:00000200:3.0:1788205263.710038:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.710043:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205263.710048:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.710051:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205263.710054:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205263.710057:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205263.710061:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205263.710064:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205263.710068:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.710070:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205263.710072:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205263.710075:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205263.710078:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205263.710080:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205263.710082:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205263.710085:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710118:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.710121:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710127:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.710128:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205263.710132:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205263.710136:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205263.710139:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205263.710141:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321622548 00000100:00000001:3.0:1788205263.710143:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321623814 00000100:00000001:3.0:1788205263.710145:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321622548 00000100:00000001:3.0:1788205263.710148:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321623814 00000100:00000001:3.0:1788205263.710150:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205263.710152:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.710154:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205263.710157:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205263/real 1788205263] req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205263.710164:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205263.710165:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205263.710167:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205263.710169:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205263.710171:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205263.710173:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205263.710176:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205263.710184:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710189:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205263.710190:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710192:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.710193:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205263.710197:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710203:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205263.710206:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710212:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205263.710215:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710216:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.710217:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710219:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205263.710232:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205263.710235:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205263.710238:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205263.710241:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205263.710244:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205263.710252:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531392:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205263.710255:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205263.710258:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710265:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205263.710266:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.710268:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205263.710270:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205263.710272:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205263.710275:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531520:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205263.710278:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205263.710280:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205263.710281:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.710283:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205263.710286:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205263.710287:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205263.710290:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205263.710293:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205263.710296:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205263.710298:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543531520, portal 10 00000100:00000001:3.0:1788205263.710300:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205263.710302:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205263.710308:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710335:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205263.710338:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205263.710340:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205263.710343:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543531520, offset 0 00000400:00000200:3.0:1788205263.710355:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205263.710359:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.710362:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205263.710365:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205263.710367:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205263.710369:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205263.710371:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205263.710374:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471501:471501:256:4294967295] 192.168.201.118@tcp LPNI seq info [471360:471360:8:4294967295] 00000400:00000200:3.0:1788205263.710378:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.710384:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205263.710388:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.710391:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205263.710395:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205263.710397:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205263.710401:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205263.710404:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205263.710407:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.710423:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205263.710426:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205263.710429:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205263.710432:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205263.710434:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205263.710436:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205263.710440:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710447:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.710450:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710461:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.710462:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205263.710465:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205263.710468:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205263.710471:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205263.710472:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321622549 00000100:00000001:3.0:1788205263.710475:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321623814 00000100:00000001:3.0:1788205263.710477:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321622549 00000100:00000001:3.0:1788205263.710479:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321623814 00000100:00000001:3.0:1788205263.710481:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205263.710483:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.710485:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205263.710488:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205263/real 1788205263] req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205263.710495:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205263.710496:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205263.710498:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205263.710500:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205263.710501:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205263.710503:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205263.710506:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205263.710513:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710519:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205263.710520:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710522:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.710523:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205263.710527:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710533:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205263.710537:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710543:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205263.710545:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710546:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.710547:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710549:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205263.710551:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205263.710554:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205263.710557:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205263.710559:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205263.710563:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205263.710570:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531520:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205263.710573:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205263.710576:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710582:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205263.710584:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.710586:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205263.710587:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205263.710589:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205263.710592:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531648:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205263.710595:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205263.710596:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205263.710598:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.710600:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205263.710603:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205263.710604:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205263.710608:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205263.710610:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205263.710613:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205263.710616:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543531648, portal 4 00000100:00000001:3.0:1788205263.710617:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205263.710619:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205263.710623:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710629:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205263.710631:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205263.710633:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205263.710635:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543531648, offset 0 00000400:00000200:3.0:1788205263.710639:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205263.710642:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.710645:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205263.710648:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205263.710650:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205263.710652:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205263.710654:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205263.710657:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471502:471502:256:4294967295] 192.168.201.118@tcp LPNI seq info [471361:471361:8:4294967295] 00000400:00000200:3.0:1788205263.710660:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.710666:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205263.710670:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.710672:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205263.710676:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205263.710678:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205263.710682:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205263.710684:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205263.710688:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.710690:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205263.710692:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205263.710695:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205263.710698:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205263.710699:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205263.710702:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205263.710705:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710712:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.710714:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710720:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.710722:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205263.710724:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205263.710726:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205263.710729:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205263.710730:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321622558 00000100:00000001:3.0:1788205263.710733:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321623814 00000100:00000001:3.0:1788205263.710734:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321622558 00000100:00000001:3.0:1788205263.710736:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321623814 00000100:00000001:3.0:1788205263.710738:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205263.710740:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.710741:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205263.710745:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205263/real 1788205263] req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205263.710751:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205263.710753:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205263.710754:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205263.710756:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205263.710758:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205263.710760:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205263.710763:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205263.710770:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710776:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205263.710777:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710779:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.710780:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205263.710783:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710789:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205263.710793:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710798:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205263.710801:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710802:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.710803:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.710805:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205263.710807:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205263.710809:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205263.710813:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205263.710815:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205263.710818:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205263.710825:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531648:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205263.710829:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205263.710833:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710838:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205263.710839:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.710841:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205263.710843:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205263.710845:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205263.710847:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531776:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205263.710850:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205263.710852:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205263.710853:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205263.710855:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205263.710858:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205263.710860:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205263.710863:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205263.710865:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205263.710868:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205263.710870:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543531776, portal 4 00000100:00000001:3.0:1788205263.710872:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205263.710873:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205263.710892:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710898:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205263.710900:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205263.710902:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205263.710904:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543531776, offset 0 00000400:00000200:3.0:1788205263.710908:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205263.710911:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.710914:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205263.710916:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205263.710918:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205263.710921:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205263.710923:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205263.710926:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471503:471503:256:4294967295] 192.168.201.118@tcp LPNI seq info [471362:471362:8:4294967295] 00000400:00000200:3.0:1788205263.710929:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205263.710934:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205263.710938:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.710941:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205263.710944:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205263.710947:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000200:3.1:1788205263.710951:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205263.710953:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 37 secs later) 00000800:00000100:3.0:1788205263.710957:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205263.710959:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205263.710961:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205263.710964:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205263.710966:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205263.710968:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205263.710970:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205263.710973:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710980:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.710982:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205288 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.710988:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.710990:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205263.710992:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205263.710995:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205263.710997:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205263.710999:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321622581 00000100:00000001:3.0:1788205263.711001:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321623814 00000100:00000001:3.0:1788205263.711003:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321622581 00000100:00000001:3.0:1788205263.711005:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321623814 00000100:00000001:3.0:1788205263.711007:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205263.711009:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.711011:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205263.711014:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205263/real 1788205263] req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205263.711021:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205263.711022:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205263.711024:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205263.711026:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205263.711028:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205263.711029:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205263.711032:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205263.711040:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.711045:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205263.711046:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711048:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.711049:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205263.711053:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.711059:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205263.711063:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.711069:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205263.711072:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711073:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205263.711074:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711076:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205263.711078:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205263.711080:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205263.711083:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205263.711085:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205263.711088:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205263.711112:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543531776:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205263.711116:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711118:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.711121:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543531264/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.711127:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205263.711130:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205263.711132:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788205263.711135:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205263.711137:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205263.711138:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205263.711141:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205263.711143:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba800. 02000000:00000001:3.0:1788205263.711146:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205263.711148:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205263.711149:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205263.711156:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205263.711157:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711159:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.711162:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543531392/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.711168:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205263.711169:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205263.711170:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205263.711172:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205263.711173:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205263.711174:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205263.711176:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205263.711178:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb400. 02000000:00000001:3.0:1788205263.711180:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205263.711181:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205263.711182:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788205263.711186:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205263.711188:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711189:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.711192:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543531520/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.711198:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205263.711199:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205263.711200:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205263.711203:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205263.711204:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205263.711205:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205263.711207:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205263.711208:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8c00. 02000000:00000001:3.0:1788205263.711210:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205263.711211:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205263.711212:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205263.711214:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205263.711215:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711217:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.711220:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543531648/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.711240:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205263.711241:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205263.711242:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205263.711244:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205263.711246:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205263.711247:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205263.711249:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205263.711251:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9400. 02000000:00000001:3.0:1788205263.711253:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205263.711254:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205263.711255:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205263.711257:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205263.711258:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711260:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205263.711263:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543531776/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205288 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205263.711269:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205263.711270:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205263.711271:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205263.711273:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205263.711274:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205263.711275:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205263.711277:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205263.711278:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb000. 02000000:00000001:3.0:1788205263.711280:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205263.711281:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205263.711283:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205263.711288:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205263.711289:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711291:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205263.711308:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205263.711310:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205263.711312:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205263.711314:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205264.540197:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205264.540200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205264.540205:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205264.540206:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205264.540219:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205264.540219:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540223:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205264.540226:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205264.540226:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205264.540228:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205264.540230:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540232:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00100000:2.0:1788205264.540235:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205264.540237:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205264.540240:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205264.540240:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205264.540243:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205264.540246:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205264.540246:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205264.540246:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205264.540253:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205264.540253:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205264.540254:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205264.540259:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205264.540259:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205264.540259:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205264.540266:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205264.540266:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540268:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205264.540273:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205264.540274:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205264.540276:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540278:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205264.540280:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540282:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540284:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540289:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205264.540291:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205264.540292:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205264.540293:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540295:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205264.540297:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540299:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205264.540301:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205264.540359:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205264.540362:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205264.540365:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205265.052267:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321623814, at 4321623872 00000100:00000010:2.0:1788205265.052294:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc8a0. 00000100:00000040:3.0:1788205265.052403:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321623814, at 4321623872 00000100:00000010:3.0:1788205265.052409:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc780. 00000100:00000001:2.0:1788205265.564243:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205265.564243:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205265.564254:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205265.564254:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205265.564259:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205265.564260:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205265.564260:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205265.564267:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205265.564272:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205265.564273:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205265.564275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205265.564280:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205265.564284:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205265.564285:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205265.564286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205265.564291:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205265.564293:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205265.564294:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205265.564299:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205265.564302:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205265.564303:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205265.564308:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205265.564309:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205265.564313:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205265.564315:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205265.564318:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205265.564381:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205265.564385:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205265.564389:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205266.588205:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205266.588205:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205266.588207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205266.588213:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205266.588214:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205266.588214:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205266.588218:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205266.588218:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205266.588221:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205266.588225:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205266.588226:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205266.588226:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205266.588229:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205266.588233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205266.588234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205266.588235:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205266.588238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205266.588240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205266.588240:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205266.588241:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205266.588247:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205266.588248:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205266.588253:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205266.588253:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205266.588260:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205266.588260:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205266.588318:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205266.588322:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205266.588326:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205267.612233:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205267.612233:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205267.612243:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205267.612243:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205267.612247:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205267.612248:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205267.612252:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205267.612255:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205267.612261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205267.612263:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788205267.612264:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205267.612272:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205267.612273:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205267.612274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205267.612275:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205267.612281:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205267.612281:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205267.612282:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205267.612289:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205267.612289:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205267.612291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205267.612297:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205267.612299:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205267.612299:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205267.612306:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205267.612310:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205267.614225:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205267.614231:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205267.614235:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205268.636181:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205268.636189:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205268.636192:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205268.636198:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205268.636202:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205268.636205:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205268.636208:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205268.636211:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205268.636215:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205268.636227:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205268.636233:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205268.636238:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205268.636239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205268.636241:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205268.636242:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205268.636244:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205268.636244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.636329:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205268.636335:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205268.636337:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205268.636340:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000020:00000001:1.0:1788205268.636344:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000100:00000001:3.0:1788205268.636344:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205268.636348:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788205268.636350:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.636355:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00010000:1.0:1788205268.636356:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205268.636361:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.636366:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205268.828281:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205268.828295:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205268.828299:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205268.828303:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205268.828308:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205268.828316:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26955 00000400:00000200:0.0:1788205268.828327:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205268.828334:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205268.828337:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205268.828339:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828342:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205268.828344:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828345:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205268.828469:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205268.828478:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205268.828482:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205268.828487:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205268.828490:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205268.828494:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205268.828513:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828517:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205268.828520:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828522:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205268.828525:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205268.828529:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828532:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205268.828534:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205268.828538:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205268.828541:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828543:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205268.828546:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205268.828556:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788205268.828561:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205268.828565:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205268.828566:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828579:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205268.828582:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205268.828586:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205268.828593:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28800. 00000100:00000001:0.0:1788205268.828600:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205268.828608:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205268.828615:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205268.828618:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205268.828621:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205268.828651:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205268.828655:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205268.828659:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205268.828661:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205268.828663:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205268.828667:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205268.828670:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26955 00000400:00000200:0.0:1788205268.828679:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205268.828683:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205268.828686:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205268.828689:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828691:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205268.828693:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828694:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205268.828696:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205268.828697:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205268.828698:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205268.828700:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205268.828701:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205268.828702:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788205268.828704:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828706:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205268.828708:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1788205268.828709:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828713:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205268.828714:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1788205268.828717:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000001:3.0:1788205268.828718:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788205268.828724:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000040:3.0:1788205268.828729:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205268.828730:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205268.828734:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205268.828742:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205268.828743:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788205268.828747:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.828747:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828751:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205268.828754:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1788205268.828757:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205268.828758:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788205268.828764:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.828765:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828771:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205268.828777:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000100:00100000:3.0:1788205268.828778:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543531904:192.168.201.118@tcp:250:kworker.0 00000020:00000040:0.0:1788205268.828780:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205268.828785:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205268.828786:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788205268.828787:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828790:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205268.828791:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205268.828793:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.828793:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205268.828798:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205268.828799:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 02000000:00000001:3.0:1788205268.828803:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788205268.828804:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205268.828810:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:3.0:1788205268.828810:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205268.828815:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828821:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205268.828823:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205268.828825:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205268.828831:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205268.828833:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205268.828835:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205268.828836:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205268.828837:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205268.828848:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205268.828848:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205268.828854:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26955 00000400:00000010:3.0:1788205268.828856:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000200:0.0:1788205268.828860:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205268.828864:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205268.828866:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205268.828870:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828872:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205268.828874:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828876:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205268.828878:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205268.828879:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788205268.828881:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205268.828883:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205268.828885:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788205268.828887:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788205268.828889:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828891:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205268.828892:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828893:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205268.828895:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788205268.828897:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205268.828898:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205268.828899:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205268.828901:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205268.828902:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828903:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205268.828905:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205268.828906:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828913:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205268.828917:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788205268.828920:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205268.828922:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205268.828923:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.828925:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205268.828926:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205268.828928:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205268.828932:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000400:00000010:3.0:1788205268.828933:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000001:0.0:1788205268.828936:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205268.828943:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543531904, portal 25 00000100:00100000:0.0:1788205268.828944:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205268.828949:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205268.828952:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904118528 : -109419805433088 : ffff9c7bb75da300) 00000100:00000001:0.0:1788205268.828959:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205268.828962:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205268.828963:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.828973:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1788205268.828975:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1788205268.828979:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205268.828985:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205268.828990:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543531904, offset 0 00000100:00000001:0.0:1788205268.828991:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205268.828997:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205268.829001:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00080000:0.0:1788205268.829004:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205268.829006:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205268.829008:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205268.829011:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205268.829014:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26955 00000400:00000200:0.0:1788205268.829019:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205268.829021:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205268.829023:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205268.829027:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000200:3.0:1788205268.829027:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:0.0:1788205268.829032:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205268.829034:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.829035:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205268.829038:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205268.829039:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000400:00000200:3.0:1788205268.829039:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205268.829046:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205268.829052:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000020:00000001:0.0:1788205268.829053:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205268.829057:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000400:00000200:3.0:1788205268.829058:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000020:00000001:0.0:1788205268.829063:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788205268.829064:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000020:00000001:0.0:1788205268.829068:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788205268.829072:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471504:471504:256:4294967295] 192.168.201.118@tcp LPNI seq info [471363:471363:8:4294967295] 00000100:00000040:0.0:1788205268.829076:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.829081:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205268.829082:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:0.0:1788205268.829086:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.829131:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205268.829146:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788205268.829150:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205268.829155:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205268.829157:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205268.829166:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 02000000:00000001:0.0:1788205268.829171:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:3.0:1788205268.829176:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 02000000:00000001:0.0:1788205268.829177:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.829182:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205268.829185:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000800:00000010:3.0:1788205268.829185:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000008:00000001:0.0:1788205268.829191:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.829193:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205268.829197:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000010:0.0:1788205268.829200:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000800:00000200:3.0:1788205268.829203:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000020:00000040:0.0:1788205268.829204:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205268.829211:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205268.829213:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.1:1788205268.829215:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788205268.829217:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205268.829221:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:3.1:1788205268.829222:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 02000000:00000001:0.0:1788205268.829224:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788205268.829230:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 02000000:00000010:0.0:1788205268.829232:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000800:00000010:3.0:1788205268.829234:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000100:00000001:0.0:1788205268.829239:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205268.829247:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205268.829254:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000400:00000200:3.0:1788205268.829255:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1788205268.829258:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205268.829266:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205268.829266:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205268.829274:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000100:00000001:0.0:1788205268.829277:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205268.829278:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00080000:0.0:1788205268.829281:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205268.829286:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205268.829287:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205268.829289:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205268.829292:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1788205268.829292:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1788205268.829296:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26955 00000100:00000200:3.0:1788205268.829301:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788205268.829306:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205268.829313:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205268.829317:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205268.829318:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1788205268.829323:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000040:3.0:1788205268.829326:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205268.829327:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205268.829333:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.829338:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205268.829343:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205268.829344:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205268.829346:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000001:3.0:1788205268.829348:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1788205268.829350:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205268.829353:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205268.829355:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205268.829357:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788205268.829360:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.829361:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205268.829363:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205268.829366:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205268.829366:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1788205268.829371:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000400:00000200:3.0:1788205268.829374:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00080000:0.0:1788205268.829378:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205268.829381:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321628934 00000100:00000001:0.0:1788205268.829382:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205268.829386:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000010:3.0:1788205268.829388:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc780. 02000000:00000001:0.0:1788205268.829389:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205268.829394:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.829396:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205268.829398:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000008:00100000:0.0:1788205268.829399:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205268.829403:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.829405:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.829408:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321628934 00000100:00000010:0.0:1788205268.829409:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788205268.829414:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000010:3.0:1788205268.829415:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc060. 02000000:00000001:0.0:1788205268.829418:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205268.829420:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:0.0:1788205268.829421:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205268.829426:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205268.829426:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205268.829427:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205268.829432:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.829432:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.829437:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 02000000:00000010:0.0:1788205268.829439:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00100000:3.0:1788205268.829443:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205268/real 1788205268] req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205268.829445:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205268.829455:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788205268.829459:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205268.829462:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205268.829466:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000100:00000001:0.0:1788205268.829467:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205268.829471:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205268.829475:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205268.829483:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205268.829488:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26965) 00000400:00000200:3.0:1788205268.830740:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205268.830745:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205268.830749:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205268.830754:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205268.830764:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.830771:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205268.830773:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.830776:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.830778:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205268.830782:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.830789:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205268.830793:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.830801:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205268.830804:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.830805:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.830806:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.830808:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205268.830810:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205268.830813:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205268.830817:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205268.830820:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205268.830824:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205268.830832:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543531904:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205268.830837:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.830839:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.830842:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da300 x1875055543531904/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.830849:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205268.830851:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205268.830854:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205268.830857:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205268.830859:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205268.830861:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205268.830863:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205268.830866:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28800. 02000000:00000001:3.0:1788205268.830933:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205268.830936:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205268.830938:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da300. 00000100:00000001:3.0:1788205268.830943:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205268.830944:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.830946:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.830973:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205268.830975:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.830977:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205268.830979:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205268.830981:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205268.830986:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.830993:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205268.830995:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.830997:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205268.831000:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205268.831002:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205268.831006:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532032:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205268.831009:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205268.831013:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205268.831015:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.831017:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205268.831022:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205268.831024:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205268.831028:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205268.831031:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205268.831035:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205268.831039:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543532032, portal 10 00000100:00000001:3.0:1788205268.831041:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205268.831042:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114048 : -109419805437568 : ffff9c7bb75d9180) 00000100:00000040:3.0:1788205268.831049:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831055:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205268.831057:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205268.831059:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205268.831061:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543532032, offset 0 00000400:00000200:3.0:1788205268.831066:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205268.831071:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205268.831076:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205268.831080:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205268.831083:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205268.831087:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205268.831104:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205268.831108:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471505:471505:256:4294967295] 192.168.201.118@tcp LPNI seq info [471364:471364:8:4294967295] 00000400:00000200:3.0:1788205268.831113:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205268.831120:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205268.831125:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205268.831129:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205268.831145:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205268.831148:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205268.831153:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205268.831156:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205268.831160:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205268.831162:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205268.831165:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205268.831168:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205268.831172:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205268.831174:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205268.831176:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205268.831180:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831187:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.831190:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831196:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.831197:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205268.831201:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205268.831205:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205268.831208:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205268.831211:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321627669 00000100:00000001:3.0:1788205268.831214:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321628934 00000100:00000001:3.0:1788205268.831216:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321627669 00000100:00000001:3.0:1788205268.831218:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321628934 00000100:00000001:3.0:1788205268.831220:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205268.831222:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.831224:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205268.831227:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205268/real 1788205268] req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205268.831241:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205268.831243:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205268.831244:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205268.831247:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205268.831249:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205268.831251:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205268.831254:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205268.831262:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831269:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205268.831270:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831272:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.831273:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205268.831277:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831283:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205268.831286:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831293:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205268.831296:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831297:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.831299:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831300:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205268.831302:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205268.831305:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205268.831308:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205268.831310:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205268.831314:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205268.831322:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532032:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205268.831326:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205268.831329:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831335:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205268.831337:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.831339:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205268.831340:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205268.831342:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205268.831345:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532160:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205268.831348:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205268.831350:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205268.831352:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.831353:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205268.831356:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205268.831358:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205268.831361:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205268.831364:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205268.831366:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205268.831369:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543532160, portal 10 00000100:00000001:3.0:1788205268.831371:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205268.831372:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904123904 : -109419805427712 : ffff9c7bb75db800) 00000100:00000040:3.0:1788205268.831377:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831383:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205268.831385:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205268.831387:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205268.831389:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543532160, offset 0 00000400:00000200:3.0:1788205268.831393:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205268.831396:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205268.831399:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205268.831402:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205268.831405:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205268.831407:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205268.831409:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205268.831412:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471506:471506:256:4294967295] 192.168.201.118@tcp LPNI seq info [471365:471365:8:4294967295] 00000400:00000200:3.0:1788205268.831416:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205268.831421:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205268.831434:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205268.831441:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205268.831445:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205268.831447:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205268.831451:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205268.831454:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205268.831458:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205268.831459:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205268.831462:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205268.831465:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205268.831468:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205268.831470:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205268.831472:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205268.831476:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831483:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.831486:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831492:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.831493:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205268.831496:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205268.831499:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205268.831502:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205268.831504:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321627670 00000100:00000001:3.0:1788205268.831506:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321628934 00000100:00000001:3.0:1788205268.831508:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321627670 00000100:00000001:3.0:1788205268.831510:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321628934 00000100:00000001:3.0:1788205268.831512:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205268.831514:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.831516:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205268.831519:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205268/real 1788205268] req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205268.831526:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205268.831527:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205268.831529:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205268.831531:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205268.831532:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205268.831534:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205268.831538:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205268.831545:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831551:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205268.831553:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831554:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.831556:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205268.831559:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831566:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205268.831569:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831575:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205268.831578:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831579:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.831580:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831582:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205268.831584:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205268.831587:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205268.831590:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205268.831592:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205268.831595:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205268.831603:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532160:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205268.831606:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205268.831609:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831616:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205268.831617:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.831620:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205268.831621:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205268.831624:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205268.831626:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532288:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205268.831629:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205268.831632:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205268.831633:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.831635:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205268.831638:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205268.831640:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205268.831643:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205268.831645:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205268.831648:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205268.831651:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543532288, portal 4 00000100:00000001:3.0:1788205268.831653:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205268.831654:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904113152 : -109419805438464 : ffff9c7bb75d8e00) 00000100:00000040:3.0:1788205268.831659:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831665:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205268.831667:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205268.831669:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205268.831671:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543532288, offset 0 00000400:00000200:3.0:1788205268.831675:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205268.831678:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205268.831682:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205268.831684:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205268.831687:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205268.831689:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205268.831691:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205268.831694:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471507:471507:256:4294967295] 192.168.201.118@tcp LPNI seq info [471366:471366:8:4294967295] 00000400:00000200:3.0:1788205268.831698:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205268.831703:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205268.831707:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205268.831710:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205268.831714:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205268.831716:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205268.831721:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205268.831723:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205268.831727:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205268.831729:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205268.831731:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205268.831734:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205268.831737:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205268.831739:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205268.831741:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205268.831745:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831757:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.831760:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831766:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.831767:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205268.831770:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205268.831773:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205268.831776:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205268.831778:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321627679 00000100:00000001:3.0:1788205268.831780:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321628934 00000100:00000001:3.0:1788205268.831782:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321627679 00000100:00000001:3.0:1788205268.831784:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321628934 00000100:00000001:3.0:1788205268.831786:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205268.831788:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.831790:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205268.831793:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205268/real 1788205268] req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205268.831801:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205268.831802:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205268.831803:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205268.831806:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205268.831807:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205268.831809:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205268.831812:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205268.831820:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831826:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205268.831827:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831829:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.831830:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205268.831833:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831840:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205268.831843:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831849:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205268.831852:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831853:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.831854:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.831856:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205268.831858:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205268.831861:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205268.831864:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205268.831866:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205268.831870:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205268.831943:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532288:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205268.831948:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205268.831951:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.831957:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205268.831959:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.831960:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205268.831962:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205268.831964:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205268.831967:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532416:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205268.831969:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205268.831971:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205268.831973:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205268.831975:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205268.831978:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205268.831979:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205268.831982:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205268.831985:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205268.831988:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205268.831991:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543532416, portal 4 00000100:00000001:3.0:1788205268.831993:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205268.831994:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114944 : -109419805436672 : ffff9c7bb75d9500) 00000100:00000040:3.0:1788205268.832000:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832006:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205268.832008:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205268.832009:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205268.832011:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543532416, offset 0 00000400:00000200:3.0:1788205268.832015:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205268.832019:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205268.832022:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205268.832024:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205268.832027:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205268.832029:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205268.832031:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205268.832034:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471508:471508:256:4294967295] 192.168.201.118@tcp LPNI seq info [471367:471367:8:4294967295] 00000400:00000200:3.0:1788205268.832038:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205268.832044:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205268.832048:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205268.832050:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205268.832054:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205268.832057:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000200:3.1:1788205268.832061:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205268.832063:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 32 secs later) 00000800:00000100:3.0:1788205268.832067:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205268.832069:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205268.832072:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205268.832075:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205268.832077:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205268.832079:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205268.832081:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205268.832085:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832109:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.832112:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205293 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832118:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.832120:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205268.832123:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205268.832125:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205268.832128:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205268.832130:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321627703 00000100:00000001:3.0:1788205268.832143:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321628934 00000100:00000001:3.0:1788205268.832145:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321627703 00000100:00000001:3.0:1788205268.832147:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321628934 00000100:00000001:3.0:1788205268.832150:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205268.832151:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.832153:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205268.832157:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205268/real 1788205268] req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205268.832164:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205268.832166:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205268.832168:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205268.832170:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205268.832171:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205268.832174:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205268.832176:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205268.832184:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832190:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205268.832191:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832193:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.832194:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205268.832198:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832204:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205268.832208:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832214:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205268.832217:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832218:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205268.832220:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832221:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205268.832223:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205268.832225:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205268.832229:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205268.832231:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205268.832235:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205268.832248:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532416:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205268.832263:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832265:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.832269:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9180 x1875055543532032/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832276:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205268.832277:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205268.832279:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205268.832282:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205268.832283:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205268.832284:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205268.832286:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205268.832288:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:3.0:1788205268.832290:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205268.832291:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205268.832293:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9180. 00000100:00000001:3.0:1788205268.832295:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205268.832296:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832298:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.832300:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db800 x1875055543532160/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832306:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205268.832308:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205268.832309:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205268.832311:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205268.832312:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205268.832313:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205268.832315:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205268.832316:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a800. 02000000:00000001:3.0:1788205268.832318:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205268.832319:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205268.832321:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db800. 00000100:00000001:3.0:1788205268.832323:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205268.832325:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832326:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.832329:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8e00 x1875055543532288/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832334:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205268.832335:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205268.832337:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205268.832339:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205268.832340:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205268.832341:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205268.832342:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205268.832343:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2ac00. 02000000:00000001:3.0:1788205268.832345:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205268.832347:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205268.832348:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8e00. 00000100:00000001:3.0:1788205268.832350:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205268.832351:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832353:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205268.832356:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9500 x1875055543532416/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205293 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205268.832361:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205268.832362:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205268.832364:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205268.832366:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205268.832367:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205268.832368:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205268.832369:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205268.832371:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28c00. 02000000:00000001:3.0:1788205268.832372:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205268.832374:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205268.832375:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9500. 00000100:00000001:3.0:1788205268.832377:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205268.832378:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832379:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205268.832384:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205268.832402:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205268.832404:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205268.832406:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205269.660273:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205269.660274:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205269.660275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205269.660287:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205269.660288:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205269.660289:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205269.660292:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205269.660293:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205269.660295:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205269.660300:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205269.660301:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788205269.660305:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205269.660306:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:3.0:1788205269.660307:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205269.660312:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205269.660314:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205269.660316:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205269.660318:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205269.660319:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205269.660323:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205269.660324:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205269.660324:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205269.660327:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205269.660338:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205269.660342:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205269.660343:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205269.660347:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205269.660351:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205269.660355:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205269.660358:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205269.660364:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205269.660366:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205269.660478:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205269.660483:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205269.660488:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205270.172177:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321628934, at 4321628992 00000100:00000040:3.0:1788205270.172177:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321628934, at 4321628992 00000100:00000010:2.0:1788205270.172194:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc780. 00000100:00000010:3.0:1788205270.172199:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc060. 00000100:00000001:2.0:1788205270.684256:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205270.684267:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205270.684269:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205270.684276:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205270.684283:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205270.684285:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205270.684288:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205270.684294:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205270.684298:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205270.684302:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205270.684303:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205270.684309:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205270.684317:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:0.0:1788205270.684322:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788205270.684327:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205270.684331:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788205270.684335:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788205270.684340:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205270.684369:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205270.684373:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205270.684379:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205270.684381:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205270.684385:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205270.684386:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205270.684389:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205270.684391:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205270.684447:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205270.684452:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205270.684456:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205271.708214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205271.708222:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205271.708224:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205271.708230:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205271.708234:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205271.708239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205271.708242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205271.708246:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205271.708248:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205271.708286:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205271.708293:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205271.708298:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205271.708300:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205271.708303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205271.708304:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205271.708307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205271.708308:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205271.708375:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205271.708380:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205271.708384:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205271.708516:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205271.708522:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205271.708524:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205271.708528:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205271.708531:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205271.708534:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205271.708537:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205271.708539:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205271.708542:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205272.732191:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205272.732197:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205272.732200:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205272.732204:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205272.732204:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205272.732211:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205272.732213:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205272.732214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205272.732215:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205272.732217:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.732220:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205272.732220:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205272.732222:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205272.732225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205272.732225:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205272.732226:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205272.732229:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788205272.732231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205272.732233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205272.732233:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.732235:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205272.732236:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205272.732240:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205272.732244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205272.732247:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205272.732249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205272.732293:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205272.732297:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205272.732299:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205273.756245:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205273.756245:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205273.756245:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205273.756257:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.756258:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205273.756259:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205273.756269:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205273.756270:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205273.756272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205273.756278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205273.756287:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205273.756287:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205273.756288:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205273.756294:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.756296:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205273.756297:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205273.756300:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205273.756307:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.756311:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205273.756312:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205273.756323:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205273.756323:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205273.756327:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.756329:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205273.756331:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.756336:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205273.756368:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205273.756371:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205273.756373:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788205273.948184:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205273.948195:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205273.948197:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205273.948200:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205273.948203:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205273.948210:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26960 00000400:00000200:0.0:1788205273.948219:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205273.948226:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205273.948229:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205273.948231:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948233:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205273.948235:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948237:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205273.948240:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205273.948241:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205273.948244:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205273.948246:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205273.948247:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205273.948249:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205273.948251:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948252:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205273.948254:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948255:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205273.948257:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205273.948259:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948260:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205273.948262:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205273.948265:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205273.948267:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948269:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205273.948271:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205273.948279:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9500. 00000020:00000040:0.0:1788205273.948281:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205273.948284:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205273.948285:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948288:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205273.948290:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205273.948293:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205273.948301:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29000. 00000100:00000001:0.0:1788205273.948307:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205273.948314:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205273.948321:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205273.948324:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205273.948327:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000eb8da790] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205273.948348:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205273.948350:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205273.948353:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205273.948354:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205273.948355:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205273.948357:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205273.948359:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26960 00000400:00000200:0.0:1788205273.948361:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205273.948362:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205273.948364:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205273.948366:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948367:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205273.948368:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948370:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205273.948371:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205273.948371:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205273.948372:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205273.948374:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205273.948375:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205273.948377:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788205273.948378:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948380:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205273.948382:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948383:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205273.948385:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788205273.948387:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948389:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205273.948390:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205273.948394:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205273.948395:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788205273.948396:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948398:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205273.948403:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788205273.948403:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000008:00000001:0.0:1788205273.948406:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.948407:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788205273.948408:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205273.948412:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8e00. 00000020:00000040:0.0:1788205273.948414:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205273.948416:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205273.948417:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205273.948418:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205273.948419:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205273.948421:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205273.948423:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205273.948427:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2bc00. 00000100:00000001:0.0:1788205273.948432:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.948436:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1788205273.948439:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205273.948440:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.948447:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788205273.948448:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205273.948451:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.948453:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1788205273.948454:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001f06ec11] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788205273.948459:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948463:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205273.948466:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788205273.948467:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543532544:192.168.201.118@tcp:250:kworker.0 00000100:00080000:0.0:1788205273.948471:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205273.948474:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205273.948476:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205273.948476:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788205273.948480:0:409030:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1788205273.948482:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788205273.948483:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26960 00000400:00000200:0.0:1788205273.948487:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000001:3.0:1788205273.948487:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1788205273.948491:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205273.948493:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1788205273.948493:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1788205273.948497:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948498:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205273.948500:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948501:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 02000000:00000010:3.0:1788205273.948501:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205273.948504:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788205273.948505:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205273.948507:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788205273.948509:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205273.948510:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205273.948513:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205273.948513:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:0.0:1788205273.948516:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000010:3.0:1788205273.948518:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000100:00000040:0.0:1788205273.948520:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948523:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205273.948524:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948525:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000010:3.0:1788205273.948526:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000020:00000040:0.0:1788205273.948527:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788205273.948531:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948532:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205273.948533:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:3.0:1788205273.948534:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543532544, portal 25 02000000:00000001:0.0:1788205273.948535:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205273.948539:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788205273.948540:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.948542:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114944 : -109419805436672 : ffff9c7bb75d9500) 00000100:00000001:0.0:1788205273.948543:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205273.948546:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205273.948549:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205273.948550:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205273.948551:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205273.948556:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db800. 00000020:00000040:0.0:1788205273.948559:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205273.948563:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205273.948564:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788205273.948565:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948568:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:3.0:1788205273.948569:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 02000000:00000001:0.0:1788205273.948570:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205273.948572:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205273.948573:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 02000000:00000010:0.0:1788205273.948576:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a400. 00000100:00000200:3.0:1788205273.948577:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543532544, offset 0 00000100:00000001:0.0:1788205273.948581:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205273.948587:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205273.948587:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948594:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205273.948596:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205273.948597:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:0.0:1788205273.948601:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000072eb16e2] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788205273.948605:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000100:00000001:0.0:1788205273.948613:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205273.948613:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1788205273.948616:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:3.0:1788205273.948619:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00080000:0.0:1788205273.948621:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205273.948623:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:3.0:1788205273.948625:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00080000:0.0:1788205273.948626:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205273.948630:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:3.0:1788205273.948630:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00080000:0.0:1788205273.948635:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26960 00000400:00000200:3.0:1788205273.948638:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471509:471509:256:4294967295] 192.168.201.118@tcp LPNI seq info [471368:471368:8:4294967295] 00000400:00000200:3.0:1788205273.948641:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205273.948651:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.948651:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000400:00000200:0.0:1788205273.948659:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000800:00000200:3.0:1788205273.948660:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00080000:0.0:1788205273.948666:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000800:00000010:3.0:1788205273.948666:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000100:00000040:0.0:1788205273.948673:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000800:00000200:3.0:1788205273.948674:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788205273.948678:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:3.0:1788205273.948679:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000100:00000040:0.0:1788205273.948685:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000800:00000200:3.1:1788205273.948687:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000100:00000001:0.0:1788205273.948688:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000800:00000200:3.1:1788205273.948692:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000020:00000001:0.0:1788205273.948693:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205273.948696:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205273.948699:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000800:00000100:3.0:1788205273.948699:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.948702:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000020:00000040:0.0:1788205273.948703:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788205273.948706:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788205273.948708:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000400:00000200:3.0:1788205273.948708:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1788205273.948712:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948715:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:3.0:1788205273.948716:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000100:00000040:0.0:1788205273.948719:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948721:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205273.948722:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000020:00000040:0.0:1788205273.948723:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000400:00000010:3.0:1788205273.948725:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00080000:0.0:1788205273.948728:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948730:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.948730:0:256132:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1788205273.948733:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:3.0:1788205273.948736:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788205273.948737:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205273.948742:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948745:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205273.948748:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:3.0:1788205273.948750:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000008:00000001:0.0:1788205273.948751:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948755:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205273.948755:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788205273.948758:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d9180. 00000020:00000040:0.0:1788205273.948762:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788205273.948764:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205273.948765:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788205273.948766:0:256132:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1788205273.948768:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948771:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205273.948772:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205273.948772:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205273.948774:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205273.948777:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 02000000:00000010:0.0:1788205273.948779:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:3.0:1788205273.948780:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321634053 00000100:00000001:0.0:1788205273.948782:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205273.948787:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc060. 00000100:00100000:0.0:1788205273.948788:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205273.948795:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205273.948797:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.948798:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000040:0.0:1788205273.948801:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e001eb7f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205273.948806:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.948806:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321634053 00000100:00080000:0.0:1788205273.948810:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:3.0:1788205273.948812:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc1e0. 00000100:00080000:0.0:1788205273.948813:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205273.948817:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205273.948818:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205273.948820:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205273.948822:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26960 00000400:00000200:0.0:1788205273.948824:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205273.948827:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205273.948827:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00080000:0.0:1788205273.948830:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205273.948833:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205273.948834:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788205273.948835:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205273.948838:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205273.948840:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.948841:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205273.948843:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1788205273.948844:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205273.948847:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788205273.948850:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000100:00100000:3.0:1788205273.948850:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205273/real 1788205273] req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205273.948854:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205273.948859:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000001:3.0:1788205273.948861:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1788205273.948863:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000400:00000001:3.0:1788205273.948864:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205273.948866:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205273.948870:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000100:00000040:0.0:1788205273.948872:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000400:00000010:3.0:1788205273.948872:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205273.948876:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1788205273.948877:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:3.0:1788205273.948881:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1788205273.948883:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948889:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000200:3.0:1788205273.948893:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1788205273.948894:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788205273.948900:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205273.948906:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.948906:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205273.948908:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.948910:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.948911:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205273.948913:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.948917:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205273.948920:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.948924:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205273.948926:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.948928:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.948929:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.948930:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205273.948932:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205273.948934:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205273.948937:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205273.948938:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205273.948941:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205273.948946:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543532544:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205273.948949:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.948951:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.948952:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9500 x1875055543532544/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.948956:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205273.948958:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205273.948959:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205273.948961:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205273.948962:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205273.948963:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205273.948964:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205273.948966:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29000. 02000000:00000001:3.0:1788205273.948968:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205273.948969:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205273.948970:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9500. 00000100:00000001:3.0:1788205273.948972:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205273.948973:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.948974:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.948980:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205273.948981:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.948983:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205273.948984:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205273.948985:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205273.948987:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.948991:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205273.948992:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.948993:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205273.948994:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205273.948996:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205273.948997:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532672:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205273.948999:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205273.949000:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205273.949001:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.949003:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205273.949005:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205273.949006:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205273.949008:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205273.949009:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205273.949011:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205273.949013:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543532672, portal 10 00000100:00000001:3.0:1788205273.949014:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205273.949015:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904113152 : -109419805438464 : ffff9c7bb75d8e00) 00000100:00000040:3.0:1788205273.949019:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949023:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205273.949025:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205273.949026:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205273.949027:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543532672, offset 0 00000400:00000200:3.0:1788205273.949029:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205273.949032:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.949034:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205273.949036:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205273.949038:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205273.949039:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205273.949041:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205273.949043:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471510:471510:256:4294967295] 192.168.201.118@tcp LPNI seq info [471369:471369:8:4294967295] 00000400:00000200:3.0:1788205273.949057:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.949061:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205273.949065:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.949067:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205273.949069:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205273.949071:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205273.949074:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205273.949076:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205273.949078:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.949079:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205273.949081:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205273.949083:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205273.949084:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205273.949086:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205273.949087:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205273.949111:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949116:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.949118:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949122:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.949123:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205273.949126:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205273.949129:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205273.949131:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205273.949132:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321632786 00000100:00000001:3.0:1788205273.949134:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321634053 00000100:00000001:3.0:1788205273.949135:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321632786 00000100:00000001:3.0:1788205273.949137:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321634053 00000100:00000001:3.0:1788205273.949138:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205273.949140:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.949141:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205273.949143:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205273/real 1788205273] req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205273.949151:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205273.949152:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205273.949154:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205273.949156:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205273.949157:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205273.949160:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205273.949162:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205273.949170:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949176:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205273.949177:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949179:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.949181:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205273.949184:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949190:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205273.949194:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949200:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205273.949202:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949204:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.949205:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949207:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205273.949209:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205273.949212:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205273.949215:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205273.949217:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205273.949221:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205273.949231:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532672:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205273.949233:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205273.949235:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949240:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205273.949241:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.949242:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205273.949243:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205273.949244:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205273.949246:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532800:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205273.949248:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205273.949249:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205273.949250:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.949251:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205273.949253:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205273.949255:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205273.949257:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205273.949259:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205273.949261:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205273.949263:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543532800, portal 10 00000100:00000001:3.0:1788205273.949264:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205273.949265:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904123904 : -109419805427712 : ffff9c7bb75db800) 00000100:00000040:3.0:1788205273.949268:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949276:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205273.949277:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205273.949278:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205273.949280:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543532800, offset 0 00000400:00000200:3.0:1788205273.949282:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205273.949284:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.949286:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205273.949288:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205273.949290:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205273.949291:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205273.949293:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205273.949295:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471511:471511:256:4294967295] 192.168.201.118@tcp LPNI seq info [471370:471370:8:4294967295] 00000400:00000200:3.0:1788205273.949297:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.949301:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205273.949304:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.949306:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205273.949308:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205273.949310:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205273.949312:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205273.949314:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205273.949316:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.949318:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205273.949319:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205273.949322:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205273.949325:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205273.949326:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205273.949328:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205273.949331:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949338:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.949341:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949353:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.949354:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205273.949358:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205273.949361:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205273.949364:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205273.949366:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321632787 00000100:00000001:3.0:1788205273.949368:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321634053 00000100:00000001:3.0:1788205273.949370:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321632787 00000100:00000001:3.0:1788205273.949372:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321634053 00000100:00000001:3.0:1788205273.949374:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205273.949376:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.949378:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205273.949382:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205273/real 1788205273] req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205273.949390:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205273.949391:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205273.949392:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205273.949395:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205273.949396:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205273.949398:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205273.949402:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205273.949411:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949418:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205273.949419:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949420:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.949422:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205273.949425:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949431:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205273.949433:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949437:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205273.949439:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949440:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.949441:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949442:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205273.949443:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205273.949445:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205273.949446:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205273.949448:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205273.949450:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205273.949460:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532800:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205273.949462:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205273.949464:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949468:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205273.949469:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.949471:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205273.949472:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205273.949473:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205273.949475:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532928:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205273.949477:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205273.949478:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205273.949479:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.949481:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205273.949483:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205273.949484:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205273.949486:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205273.949488:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205273.949490:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205273.949492:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543532928, portal 4 00000100:00000001:3.0:1788205273.949493:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205273.949494:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904114048 : -109419805437568 : ffff9c7bb75d9180) 00000100:00000040:3.0:1788205273.949497:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949501:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205273.949502:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205273.949504:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205273.949505:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543532928, offset 0 00000400:00000200:3.0:1788205273.949507:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205273.949509:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.949511:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205273.949513:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205273.949515:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205273.949516:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205273.949518:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205273.949520:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471512:471512:256:4294967295] 192.168.201.118@tcp LPNI seq info [471371:471371:8:4294967295] 00000400:00000200:3.0:1788205273.949522:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.949526:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205273.949529:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.949530:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205273.949533:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205273.949534:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205273.949537:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205273.949539:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205273.949542:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.949543:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205273.949545:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205273.949547:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205273.949549:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205273.949550:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205273.949551:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205273.949553:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949557:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.949559:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949563:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.949564:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205273.949566:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205273.949568:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205273.949569:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205273.949571:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321632796 00000100:00000001:3.0:1788205273.949572:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321634053 00000100:00000001:3.0:1788205273.949573:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321632796 00000100:00000001:3.0:1788205273.949575:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321634053 00000100:00000001:3.0:1788205273.949576:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205273.949577:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.949578:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205273.949580:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205273/real 1788205273] req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205273.949584:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205273.949584:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205273.949585:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205273.949587:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205273.949588:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205273.949589:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205273.949591:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205273.949595:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949599:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205273.949599:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949600:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.949601:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205273.949603:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949606:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205273.949608:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949611:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205273.949612:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949613:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.949614:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949615:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205273.949616:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205273.949617:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205273.949619:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205273.949620:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205273.949622:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205273.949627:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543532928:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205273.949629:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949630:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.949632:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8e00 x1875055543532672/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949636:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205273.949637:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205273.949638:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205273.949639:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205273.949640:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205273.949641:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205273.949642:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205273.949643:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2bc00. 02000000:00000001:3.0:1788205273.949645:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205273.949645:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205273.949646:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8e00. 00000100:00000001:3.0:1788205273.949648:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205273.949649:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949650:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.949652:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db800 x1875055543532800/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949656:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205273.949656:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205273.949657:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205273.949658:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205273.949659:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205273.949659:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205273.949660:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205273.949661:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a400. 02000000:00000001:3.0:1788205273.949662:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205273.949663:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205273.949663:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db800. 00000100:00000001:3.0:1788205273.949664:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205273.949665:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949666:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.949667:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d9180 x1875055543532928/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.949671:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205273.949672:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205273.949672:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205273.949674:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205273.949675:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205273.949675:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205273.949676:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205273.949677:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29800. 02000000:00000001:3.0:1788205273.949678:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205273.949679:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205273.949679:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d9180. 00000100:00000001:3.0:1788205273.949680:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205273.949681:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949682:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.949684:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205273.949685:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.949686:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205273.949687:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205273.949954:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205273.949956:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205273.949958:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.949959:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205273.949960:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205273.949961:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.949962:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205273.949965:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75da300. 00000020:00000040:0.0:1788205273.949966:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205273.949968:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205273.949969:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205273.949970:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205273.949970:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205273.949972:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205273.949974:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c29800. 00000100:00000001:0.0:1788205273.949976:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205273.949979:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205273.949983:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205273.949985:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205273.949987:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005d034904] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205273.949996:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205273.950000:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26970) 00000100:00000001:3.0:1788205273.950018:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205273.950020:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205273.950021:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205273.950024:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.950029:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205273.950030:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.950031:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205273.950032:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205273.950034:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205273.950035:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533056:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205273.950037:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205273.950038:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205273.950039:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205273.950041:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205273.950043:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205273.950060:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205273.950063:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205273.950065:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205273.950068:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205273.950070:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543533056, portal 4 00000100:00000001:3.0:1788205273.950071:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205273.950072:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904118528 : -109419805433088 : ffff9c7bb75da300) 00000100:00000040:3.0:1788205273.950075:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.950080:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205273.950081:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205273.950082:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205273.950084:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543533056, offset 0 00000400:00000200:3.0:1788205273.950086:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205273.950088:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.950105:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205273.950106:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205273.950108:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205273.950110:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205273.950111:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205273.950113:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471513:471513:256:4294967295] 192.168.201.118@tcp LPNI seq info [471372:471372:8:4294967295] 00000400:00000200:3.0:1788205273.950116:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205273.950119:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205273.950122:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.950124:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205273.950126:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205273.950128:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000200:3.1:1788205273.950131:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205273.950132:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 27 secs later) 00000800:00000100:3.0:1788205273.950135:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205273.950136:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205273.950138:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205273.950140:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205273.950141:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205273.950143:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205273.950144:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205273.950147:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.950151:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.950153:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205298 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.950157:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.950158:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205273.950160:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205273.950162:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205273.950164:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205273.950165:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321632821 00000100:00000001:3.0:1788205273.950167:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321634053 00000100:00000001:3.0:1788205273.950168:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321632821 00000100:00000001:3.0:1788205273.950169:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321634053 00000100:00000001:3.0:1788205273.950171:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205273.950172:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.950173:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205273.950175:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205273/real 1788205273] req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205273.950179:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205273.950180:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205273.950180:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205273.950182:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205273.950182:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205273.950183:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205273.950185:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205273.950190:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.950193:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205273.950194:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.950196:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.950196:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205273.950198:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.950202:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205273.950204:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.950207:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205273.950209:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.950209:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205273.950210:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.950211:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205273.950212:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205273.950213:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205273.950215:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205273.950216:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205273.950218:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205273.950222:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75da300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533056:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205273.950224:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.950225:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205273.950227:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75da300 x1875055543533056/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205298 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205273.950230:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205273.950231:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205273.950232:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205273.950233:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205273.950234:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205273.950235:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205273.950236:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205273.950237:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c29800. 02000000:00000001:3.0:1788205273.950239:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205273.950240:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205273.950240:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75da300. 00000100:00000001:3.0:1788205273.950242:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205273.950243:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.950244:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205273.950246:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205273.950247:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205273.950248:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205273.950249:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205274.780158:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205274.780168:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205274.780170:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205274.780176:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205274.780180:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205274.780182:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205274.780183:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205274.780190:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205274.780190:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205274.780199:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205274.780199:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205274.780207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205274.780213:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205274.780213:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205274.780227:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205274.780231:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205274.780233:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205274.780237:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205274.780239:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:2.0:1788205274.780240:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205274.780246:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205274.780247:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780252:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780258:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205274.780261:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780263:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780266:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780275:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205274.780278:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205274.780280:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205274.780282:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780283:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205274.780285:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780286:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780288:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205274.780293:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205274.780294:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205274.780295:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780297:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205274.780298:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780299:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205274.780301:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205274.780344:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205274.780350:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205274.780353:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205275.292153:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321634053, at 4321634112 00000100:00000010:2.0:1788205275.292166:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc060. 00000100:00000040:3.0:1788205275.292199:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321634053, at 4321634112 00000100:00000010:3.0:1788205275.292218:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc1e0. 00000100:00000001:2.0:1788205275.804186:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205275.804186:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205275.804189:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205275.804193:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205275.804194:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205275.804194:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205275.804197:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205275.804199:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205275.804200:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205275.804201:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205275.804203:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205275.804204:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205275.804205:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205275.804207:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205275.804209:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205275.804210:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205275.804211:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205275.804215:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205275.804216:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205275.804216:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205275.804218:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205275.804218:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205275.804221:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205275.804223:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205275.804225:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205275.804229:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205275.804302:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205275.804310:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205275.804314:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205276.828180:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205276.828184:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205276.828187:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205276.828190:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205276.828191:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205276.828192:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205276.828194:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205276.828195:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205276.828201:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205276.828202:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:3.0:1788205276.828202:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205276.828205:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205276.828209:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205276.828211:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205276.828213:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205276.828214:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205276.828218:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205276.828220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205276.828222:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205276.828223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205276.828228:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205276.828230:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205276.828238:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205276.828241:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205276.828243:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205276.828244:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205276.828295:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205276.828301:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205276.828306:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205277.852224:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205277.852233:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205277.852235:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205277.852240:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205277.852244:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205277.852248:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205277.852251:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205277.852255:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205277.852258:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205277.852285:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205277.852300:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205277.852302:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205277.852305:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205277.852307:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205277.852309:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205277.852312:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205277.852314:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205277.852315:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205277.852970:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205277.852975:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205277.852993:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205277.852994:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205277.852998:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205277.852999:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205277.853003:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205277.853005:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205277.853109:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205277.853114:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205277.853118:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788205278.876372:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205278.876379:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205278.876386:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205278.876387:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205278.876390:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205278.876391:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205278.876395:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205278.876396:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205278.876399:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205278.876405:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205278.876407:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205278.876412:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205278.876416:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205278.876419:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205278.876421:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205278.876424:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205278.876427:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205278.879067:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205278.879072:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205278.879076:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205278.879279:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205278.879284:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205278.879286:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205278.879289:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205278.879292:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205278.879295:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205278.879297:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205278.879300:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205278.879302:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205279.068319:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205279.068329:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205279.068332:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205279.068336:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205279.068341:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205279.068348:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26965 00000400:00000200:1.0:1788205279.068358:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205279.068365:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205279.068368:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205279.068372:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068375:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205279.068377:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068380:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205279.068384:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205279.068387:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205279.068389:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205279.068393:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205279.068395:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205279.068398:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205279.068401:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068403:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205279.068405:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068407:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205279.068409:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205279.068412:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068414:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205279.068417:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205279.068421:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205279.068424:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.068427:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205279.068430:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205279.068440:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7fb80. 00000020:00000040:1.0:1788205279.068444:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205279.068448:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205279.068449:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.068453:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205279.068455:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205279.068459:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205279.068466:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8800. 00000100:00000001:1.0:1788205279.068472:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205279.068481:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205279.068491:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205279.068495:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205279.068500:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003a500b86] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205279.068522:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205279.068526:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205279.068530:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205279.068531:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205279.068533:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205279.068536:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205279.068539:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26965 00000400:00000200:1.0:1788205279.068543:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205279.068546:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205279.068548:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205279.068552:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068554:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205279.068556:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068559:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205279.068562:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205279.068563:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205279.068565:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205279.068567:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205279.068570:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000001:3.0:1788205279.068571:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788205279.068572:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205279.068578:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205279.068579:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205279.068581:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205279.068583:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205279.068585:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068589:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205279.068593:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00000040:3.0:1788205279.068594:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205279.068599:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068604:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205279.068608:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788205279.068614:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788205279.068615:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205279.068618:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205279.068620:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.068623:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205279.068624:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1788205279.068628:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788205279.068629:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1788205279.068635:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.068636:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.068639:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.068644:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543533184:192.168.201.118@tcp:250:kworker.0 00000100:00000010:1.0:1788205279.068646:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e300. 00000020:00000040:1.0:1788205279.068651:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000100:00000001:3.0:1788205279.068653:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205279.068655:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205279.068657:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.068659:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788205279.068660:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205279.068663:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205279.068663:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205279.068666:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.068669:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205279.068673:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b8000. 02000000:00000010:3.0:1788205279.068677:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 00000100:00000001:1.0:1788205279.068678:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.068682:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205279.068687:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:3.0:1788205279.068694:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205279.068701:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000100:00000001:1.0:1788205279.068702:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205279.068707:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205279.068712:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000029d047e9] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788205279.068713:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205279.068722:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543533184, portal 25 00000100:00000001:1.0:1788205279.068724:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.068727:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788205279.068729:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205279.068731:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447299968 : -109424262251648 : ffff9c7aadb7fb80) 00000100:00080000:1.0:1788205279.068736:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205279.068739:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205279.068743:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:3.0:1788205279.068743:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205279.068747:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205279.068753:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26965 00000100:00000001:3.0:1788205279.068760:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1788205279.068761:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000040:3.0:1788205279.068767:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205279.068768:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205279.068773:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000400:00000010:3.0:1788205279.068773:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000040:1.0:1788205279.068778:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000200:3.0:1788205279.068780:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543533184, offset 0 00000100:00000001:1.0:1788205279.068782:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205279.068786:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068789:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205279.068791:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000020:00000001:1.0:1788205279.068792:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205279.068795:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205279.068796:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205279.068798:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205279.068802:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000400:00000200:3.0:1788205279.068802:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000001:1.0:1788205279.068806:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205279.068811:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068812:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205279.068814:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068816:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205279.068818:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205279.068821:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068824:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205279.068827:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:3.0:1788205279.068827:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 02000000:00000001:1.0:1788205279.068833:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1788205279.068836:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1788205279.068838:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.068841:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000400:00000200:3.0:1788205279.068843:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000008:00100000:1.0:1788205279.068845:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205279.068850:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205279.068850:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1788205279.068855:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205279.068857:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000100:00000010:1.0:1788205279.068862:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 00000020:00000040:1.0:1788205279.068867:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000400:00000200:3.0:1788205279.068867:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471514:471514:256:4294967295] 192.168.201.118@tcp LPNI seq info [471373:471373:8:4294967295] 02000000:00000001:1.0:1788205279.068873:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205279.068875:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205279.068879:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205279.068880:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205279.068883:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205279.068885:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205279.068890:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9c00. 00000400:00000200:3.0:1788205279.068896:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000100:00000001:1.0:1788205279.068897:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205279.068905:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:3.0:1788205279.068909:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000100:00000001:1.0:1788205279.068919:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000010:3.0:1788205279.068919:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000100:00000001:1.0:1788205279.068922:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1788205279.068929:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205279.068933:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205279.068938:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205279.068940:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000100:00000040:1.0:1788205279.068944:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1788205279.068945:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.068948:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205279.068978:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205279.068985:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205279.068992:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000100:00000001:1.0:1788205279.068993:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205279.068995:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00080000:1.0:1788205279.068997:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205279.069002:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205279.069003:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1788205279.069004:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205279.069007:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205279.069012:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:3.0:1788205279.069012:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788205279.069018:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26965 00000400:00000200:1.0:1788205279.069023:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205279.069030:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205279.069034:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205279.069036:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205279.069038:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.069041:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205279.069045:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205279.069046:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.069050:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205279.069055:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205279.069058:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205279.069061:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205279.069064:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000100:00000001:3.0:1788205279.069064:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.069069:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1788205279.069070:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205279.069073:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205279.069077:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205279.069078:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.069080:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205279.069084:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.069088:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000400:00000200:3.0:1788205279.069088:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000020:00000040:1.0:1788205279.069121:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205279.069129:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205279.069131:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321639174 00000100:00000001:1.0:1788205279.069132:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205279.069137:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205279.069141:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1788205279.069142:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc1e0. 02000000:00000001:1.0:1788205279.069144:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.069148:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205279.069150:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000100:00000001:3.0:1788205279.069152:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000008:00000001:1.0:1788205279.069153:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.069157:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205279.069161:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205279.069162:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321639174 00000020:00000040:1.0:1788205279.069164:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205279.069168:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205279.069170:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205279.069170:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc5a0. 00000100:00000001:1.0:1788205279.069175:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788205279.069176:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 02000000:00000001:1.0:1788205279.069180:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788205279.069182:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205279.069183:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.069188:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205279.069191:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9800. 00000100:00000001:3.0:1788205279.069193:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788205279.069197:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.069200:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205279/real 1788205279] req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205279.069206:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205279.069226:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205279.069231:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788205279.069236:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1788205279.069238:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205279.069239:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205279.069243:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205279.069248:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000100:00000001:1.0:1788205279.069252:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1788205279.069253:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00080000:1.0:1788205279.069258:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788205279.069259:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:1.0:1788205279.069264:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000200:3.0:1788205279.069267:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205279.069268:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205279.069270:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205279.069276:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205279.069282:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26965 00000400:00000200:1.0:1788205279.069289:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000200:3.0:1788205279.069289:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788205279.069297:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00000001:3.0:1788205279.069301:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1788205279.069302:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205279.069303:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.069306:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1788205279.069307:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205279.069309:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205279.069313:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788205279.069316:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1788205279.069319:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205279.069324:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000001:3.0:1788205279.069329:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788205279.069330:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205279.069333:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000100:00000040:3.0:1788205279.069335:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205279.069336:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205279.069340:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205279.069344:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205279.069346:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:1.0:1788205279.069348:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205279.069350:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.069353:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1788205279.069355:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205279.069356:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205279.069361:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788205279.069361:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:1.0:1788205279.069365:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205279.069365:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205279.069369:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788205279.069370:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205279.069374:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205279.069377:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00080000:1.0:1788205279.069380:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205279.069381:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1788205279.069387:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205279.069390:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:3.0:1788205279.069391:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788205279.069393:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205279.069398:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.069401:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205279.069404:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205279.069408:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.069411:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.069415:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543533184:192.168.201.118@tcp:250:kworker.0 00000100:00000010:1.0:1788205279.069416:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205279.069421:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000100:00000001:3.0:1788205279.069423:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788205279.069425:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205279.069427:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.069428:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788205279.069429:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205279.069432:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205279.069434:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205279.069436:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7fb80 x1875055543533184/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788205279.069438:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb800. 00000100:00000001:1.0:1788205279.069443:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.069450:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1788205279.069451:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788205279.069454:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205279.069459:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 00000100:00000001:1.0:1788205279.069462:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788205279.069465:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1788205279.069466:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788205279.069470:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000100:00000040:1.0:1788205279.069474:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1788205279.069475:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205279.069481:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205279.069485:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8800. 00000100:00000001:1.0:1788205279.069488:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.069491:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205279.069495:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:1.0:1788205279.069496:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26975) 00000100:00000010:3.0:1788205279.069498:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7fb80. 00000100:00000001:3.0:1788205279.069507:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205279.069508:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.069510:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.069519:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205279.069521:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.069524:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205279.069526:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205279.069527:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205279.069531:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069537:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205279.069539:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.069541:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205279.069543:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205279.069545:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.069548:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205279.069552:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205279.069554:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205279.069556:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.069558:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205279.069561:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205279.069563:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205279.069567:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205279.069569:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205279.069574:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205279.069577:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543533312, portal 10 00000100:00000001:3.0:1788205279.069578:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205279.069579:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447293696 : -109424262257920 : ffff9c7aadb7e300) 00000100:00000040:3.0:1788205279.069586:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069592:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205279.069594:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205279.069596:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205279.069599:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543533312, offset 0 00000400:00000200:3.0:1788205279.069603:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205279.069606:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205279.069610:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205279.069613:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205279.069616:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205279.069619:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205279.069621:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205279.069624:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471515:471515:256:4294967295] 192.168.201.118@tcp LPNI seq info [471374:471374:8:4294967295] 00000400:00000200:3.0:1788205279.069628:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205279.069635:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205279.069639:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.069643:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205279.069647:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205279.069650:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205279.069655:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205279.069657:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205279.069661:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.069663:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205279.069667:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205279.069671:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205279.069674:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205279.069675:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205279.069677:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205279.069681:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069688:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205279.069690:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069696:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.069697:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205279.069700:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205279.069704:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205279.069707:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205279.069709:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321637907 00000100:00000001:3.0:1788205279.069712:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321639174 00000100:00000001:3.0:1788205279.069714:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321637907 00000100:00000001:3.0:1788205279.069716:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321639174 00000100:00000001:3.0:1788205279.069718:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205279.069720:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.069722:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205279.069726:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205279/real 1788205279] req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205279.069733:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205279.069734:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205279.069736:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205279.069738:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205279.069740:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205279.069742:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205279.069745:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205279.069752:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069759:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205279.069760:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.069761:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205279.069762:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205279.069766:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069773:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205279.069776:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069782:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205279.069784:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.069786:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205279.069787:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.069788:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205279.069791:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205279.069794:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205279.069797:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205279.069798:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205279.069802:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205279.069810:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533312:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205279.069813:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205279.069816:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069822:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205279.069823:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.069825:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205279.069827:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205279.069829:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.069832:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205279.069835:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205279.069837:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205279.069839:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.069841:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205279.069843:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205279.069845:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205279.069848:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205279.069850:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205279.069853:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205279.069856:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543533440, portal 10 00000100:00000001:3.0:1788205279.069858:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205279.069859:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205279.069865:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069870:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205279.069873:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205279.069875:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205279.069877:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543533440, offset 0 00000400:00000200:3.0:1788205279.069881:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205279.069884:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205279.069888:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205279.069890:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205279.069893:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205279.069895:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205279.069897:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205279.069900:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471516:471516:256:4294967295] 192.168.201.118@tcp LPNI seq info [471375:471375:8:4294967295] 00000400:00000200:3.0:1788205279.069903:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205279.069909:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205279.069913:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.069917:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205279.069921:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205279.069923:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205279.069927:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205279.069929:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205279.069933:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.069934:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205279.069937:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205279.069940:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205279.069943:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205279.069945:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205279.069947:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205279.069950:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069972:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205279.069975:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.069981:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.069982:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205279.069986:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205279.069988:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205279.069991:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205279.069993:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321637908 00000100:00000001:3.0:1788205279.069996:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321639174 00000100:00000001:3.0:1788205279.069998:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321637908 00000100:00000001:3.0:1788205279.070000:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321639174 00000100:00000001:3.0:1788205279.070002:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205279.070004:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.070006:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205279.070010:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205279/real 1788205279] req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205279.070018:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205279.070019:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205279.070021:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205279.070023:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205279.070025:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205279.070027:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205279.070030:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205279.070038:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070045:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205279.070046:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070048:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205279.070049:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205279.070052:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070059:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205279.070062:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070068:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205279.070071:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070072:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205279.070074:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070076:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205279.070079:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205279.070081:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205279.070084:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205279.070086:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205279.070114:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205279.070122:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533440:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205279.070126:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205279.070129:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070136:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205279.070137:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.070139:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205279.070141:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205279.070143:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.070146:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205279.070149:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205279.070151:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205279.070152:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.070154:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205279.070157:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205279.070159:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205279.070163:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205279.070165:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205279.070169:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205279.070171:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543533568, portal 4 00000100:00000001:3.0:1788205279.070173:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205279.070174:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205279.070179:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070186:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205279.070188:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205279.070191:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205279.070193:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543533568, offset 0 00000400:00000200:3.0:1788205279.070197:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205279.070201:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205279.070205:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205279.070208:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205279.070210:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205279.070212:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205279.070215:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205279.070218:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471517:471517:256:4294967295] 192.168.201.118@tcp LPNI seq info [471376:471376:8:4294967295] 00000400:00000200:3.0:1788205279.070222:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205279.070228:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205279.070232:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.070236:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205279.070240:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205279.070243:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205279.070246:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205279.070249:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205279.070253:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.070255:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205279.070257:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205279.070261:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205279.070263:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205279.070265:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205279.070267:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205279.070270:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070278:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205279.070281:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070292:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.070293:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205279.070297:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205279.070299:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205279.070303:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205279.070305:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321637918 00000100:00000001:3.0:1788205279.070307:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321639174 00000100:00000001:3.0:1788205279.070309:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321637918 00000100:00000001:3.0:1788205279.070311:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321639174 00000100:00000001:3.0:1788205279.070313:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205279.070315:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.070316:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205279.070320:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205279/real 1788205279] req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205279.070328:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205279.070329:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205279.070331:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205279.070333:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205279.070335:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205279.070337:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205279.070340:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205279.070348:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070353:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205279.070355:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070356:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205279.070358:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205279.070361:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070368:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205279.070372:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070378:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205279.070380:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070382:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205279.070383:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070385:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205279.070387:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205279.070389:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205279.070392:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205279.070394:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205279.070398:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205279.070405:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533568:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205279.070409:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205279.070412:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070418:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205279.070419:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.070422:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205279.070423:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205279.070426:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205279.070428:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205279.070432:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205279.070434:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205279.070436:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205279.070437:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205279.070440:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205279.070443:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205279.070446:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205279.070449:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205279.070451:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205279.070454:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543533696, portal 4 00000100:00000001:3.0:1788205279.070456:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205279.070457:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205279.070462:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070468:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205279.070470:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205279.070472:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205279.070474:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543533696, offset 0 00000400:00000200:3.0:1788205279.070478:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205279.070482:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205279.070486:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205279.070488:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205279.070491:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205279.070493:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205279.070495:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205279.070498:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471518:471518:256:4294967295] 192.168.201.118@tcp LPNI seq info [471377:471377:8:4294967295] 00000400:00000200:3.0:1788205279.070502:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205279.070508:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205279.070512:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.070515:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205279.070519:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205279.070522:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000200:3.1:1788205279.070525:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205279.070528:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 22 secs later) 00000800:00000100:3.0:1788205279.070531:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205279.070533:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205279.070536:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205279.070539:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205279.070541:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205279.070543:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205279.070545:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205279.070548:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070555:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205279.070558:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205304 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070565:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.070566:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205279.070569:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205279.070572:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205279.070575:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205279.070577:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321637941 00000100:00000001:3.0:1788205279.070579:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321639174 00000100:00000001:3.0:1788205279.070581:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321637941 00000100:00000001:3.0:1788205279.070583:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321639174 00000100:00000001:3.0:1788205279.070585:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205279.070587:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.070589:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205279.070593:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205279/real 1788205279] req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205279.070601:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205279.070602:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205279.070604:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205279.070606:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205279.070607:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205279.070610:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205279.070613:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205279.070621:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070627:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205279.070628:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070630:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205279.070631:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205279.070634:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070641:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205279.070644:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070651:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205279.070653:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070654:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205279.070656:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070658:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205279.070659:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205279.070662:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205279.070665:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205279.070667:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205279.070671:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205279.070678:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533696:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205279.070682:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070684:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205279.070688:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e300 x1875055543533312/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070694:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205279.070695:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205279.070697:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205279.070700:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205279.070701:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205279.070702:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205279.070704:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205279.070706:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b8000. 02000000:00000001:3.0:1788205279.070708:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205279.070710:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205279.070711:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e300. 00000100:00000001:3.0:1788205279.070718:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205279.070719:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070720:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205279.070723:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543533440/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070729:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205279.070730:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205279.070732:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205279.070734:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205279.070735:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205279.070736:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205279.070737:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205279.070739:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:3.0:1788205279.070741:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205279.070742:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205279.070743:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205279.070745:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205279.070747:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070748:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205279.070751:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543533568/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070757:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205279.070759:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205279.070760:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205279.070762:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205279.070763:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205279.070764:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205279.070766:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205279.070767:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:3.0:1788205279.070769:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205279.070770:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205279.070772:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205279.070776:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205279.070777:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070779:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205279.070782:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543533696/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205304 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205279.070787:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205279.070788:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205279.070790:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205279.070791:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205279.070793:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205279.070794:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205279.070795:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205279.070796:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788205279.070798:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205279.070799:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205279.070800:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205279.070802:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205279.070803:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070805:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.070809:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205279.070810:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.070812:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205279.070813:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.900223:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205279.900223:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205279.900231:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205279.900233:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.900238:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788205279.900239:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205279.900241:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205279.900245:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788205279.900246:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:3.0:1788205279.900250:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205279.900251:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205279.900255:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205279.900256:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205279.900258:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205279.900261:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205279.900263:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205279.900265:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205279.900269:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205279.900309:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205279.900314:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205279.900316:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205279.900319:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205279.900323:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205279.900325:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205279.900327:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205279.900329:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205279.900333:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205279.900334:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205279.900337:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205279.900339:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205279.900343:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205279.900344:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205279.902169:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205279.902177:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205279.902182:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1788205280.412963:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321639174, at 4321639232 00000100:00000010:2.0:1788205280.412985:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc1e0. 00000100:00000040:3.0:1788205280.413356:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321639174, at 4321639233 00000100:00000010:3.0:1788205280.413364:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc5a0. 00000100:00000001:3.0:1788205280.924255:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205280.924261:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205280.924269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205280.924272:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205280.924280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205280.924282:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205280.924285:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205280.924286:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205280.924291:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205280.924292:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205280.925047:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205280.925053:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205280.925057:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205280.925148:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205280.925157:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205280.925159:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205280.925165:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205280.925168:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205280.925172:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205280.925175:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205280.925178:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205280.925181:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205280.926052:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205280.926058:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205280.926062:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205280.926066:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205280.926069:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205280.926072:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205280.926075:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205281.948256:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205281.948257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205281.948257:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205281.948268:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205281.948272:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205281.948273:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205281.948279:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205281.948281:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205281.948283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205281.948287:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205281.948287:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00100000:2.0:1788205281.948294:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205281.948296:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205281.948296:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205281.948300:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205281.948301:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205281.948305:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205281.948305:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205281.948310:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205281.948323:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205281.948325:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205281.948326:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205281.948338:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205281.948338:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205281.948350:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205281.948351:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205281.948495:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205281.948502:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205281.948506:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205282.972230:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205282.972247:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205282.972248:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205282.972255:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205282.972259:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205282.972264:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205282.972267:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205282.972271:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205282.972275:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205282.972307:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205282.972314:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205282.972321:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205282.972322:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205282.972336:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205282.972338:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205282.972342:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205282.972343:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205282.972441:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205282.972445:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205282.972449:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205282.974130:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205282.974138:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205282.974140:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205282.974146:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205282.974149:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205282.974153:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205282.974156:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205282.974159:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205282.974162:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205283.996242:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205283.996248:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205283.996255:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205283.996256:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205283.996259:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205283.996260:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205283.996262:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:2.0:1788205283.996269:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205283.996269:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205283.996270:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205283.996277:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205283.996277:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205283.996280:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205283.996283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205283.996283:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205283.996284:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205283.996290:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205283.996291:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205283.996292:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205283.996294:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205283.996296:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205283.996296:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205283.996299:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205283.996302:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205283.996303:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205283.996314:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205283.998171:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205283.998178:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205283.998184:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788205284.188245:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205284.188258:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788205284.188261:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205284.188266:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205284.188272:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205284.188280:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26970 00000400:00000200:1.0:1788205284.188291:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205284.188299:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205284.188303:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205284.188306:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188309:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188313:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188315:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205284.188320:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205284.188322:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:1.0:1788205284.188325:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205284.188329:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:1.0:1788205284.188332:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:1.0:1788205284.188334:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:1.0:1788205284.188337:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188339:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188342:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188344:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205284.188348:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:1.0:1788205284.188351:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188354:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205284.188357:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205284.188361:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205284.188365:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188368:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:1.0:1788205284.188371:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205284.188382:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d880. 00000020:00000040:1.0:1788205284.188386:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:1.0:1788205284.188391:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205284.188393:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188397:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205284.188401:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205284.188406:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205284.188415:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bbc00. 00000100:00000001:1.0:1788205284.188424:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205284.188434:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205284.188445:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205284.188451:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205284.188456:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008dba9718] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205284.188484:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205284.188487:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205284.188492:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788205284.188493:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205284.188495:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205284.188498:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205284.188501:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26970 00000400:00000200:1.0:1788205284.188505:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205284.188508:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205284.188510:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205284.188513:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188516:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188518:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188520:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205284.188522:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205284.188524:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:1.0:1788205284.188526:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205284.188528:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:1.0:1788205284.188530:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:1.0:1788205284.188532:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:1.0:1788205284.188534:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188536:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188538:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188540:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205284.188542:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:1.0:1788205284.188545:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188547:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205284.188549:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205284.188555:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205284.188557:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188558:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205284.188563:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205284.188567:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188569:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205284.188572:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7f480. 00000020:00000040:1.0:1788205284.188575:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205284.188577:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205284.188579:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188581:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205284.188582:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205284.188584:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205284.188587:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba400. 00000100:00000001:1.0:1788205284.188590:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205284.188596:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205284.188604:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205284.188605:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205284.188609:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a2a233e4] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205284.188617:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205284.188620:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205284.188624:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1788205284.188626:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205284.188627:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205284.188630:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205284.188632:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26970 00000400:00000200:1.0:1788205284.188636:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205284.188639:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205284.188642:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205284.188644:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188646:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188648:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188651:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205284.188653:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205284.188654:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:1.0:1788205284.188656:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205284.188658:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:1.0:1788205284.188660:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:1.0:1788205284.188662:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:1.0:1788205284.188665:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188667:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188669:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188670:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205284.188672:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:1.0:1788205284.188675:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188677:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205284.188678:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205284.188681:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205284.188683:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188685:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205284.188687:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788205284.188700:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188702:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205284.188707:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7c700. 00000020:00000040:1.0:1788205284.188709:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:1.0:1788205284.188712:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205284.188713:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188715:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205284.188716:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205284.188718:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205284.188721:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9ba000. 00000100:00000001:1.0:1788205284.188725:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205284.188729:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205284.188743:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205284.188745:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205284.188756:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0e4778a] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205284.188764:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205284.188767:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205284.188770:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788205284.188772:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205284.188773:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205284.188776:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205284.188779:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26970 00000400:00000200:1.0:1788205284.188783:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205284.188786:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205284.188789:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1788205284.188792:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188794:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188796:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188798:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:1.0:1788205284.188801:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205284.188802:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:1.0:1788205284.188804:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205284.188806:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:1.0:1788205284.188808:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:1.0:1788205284.188810:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:1.0:1788205284.188813:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188814:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188817:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188819:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:1.0:1788205284.188822:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:1.0:1788205284.188825:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188827:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205284.188829:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788205284.188831:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205284.188833:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188835:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:1.0:1788205284.188837:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:1.0:1788205284.188840:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188842:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1788205284.188845:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7d180. 00000020:00000040:1.0:1788205284.188847:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205284.188849:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205284.188851:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.188852:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205284.188854:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205284.188855:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205284.188858:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9b9000. 00000100:00000001:1.0:1788205284.188877:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205284.188882:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788205284.188889:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205284.188891:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205284.188895:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000018ed9952] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205284.188903:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788205284.188905:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788205284.188909:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788205284.188911:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788205284.188913:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788205284.188916:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1788205284.188919:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26970 00000400:00000200:1.0:1788205284.188923:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205284.188924:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1788205284.188927:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:1.0:1788205284.188930:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788205284.188932:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1788205284.188935:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205284.188937:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788205284.188938:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1788205284.188941:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188944:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00000040:3.0:1788205284.188947:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788205284.188948:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:1.0:1788205284.188951:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:1.0:1788205284.188955:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:1.0:1788205284.188958:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:1.0:1788205284.188964:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205284.188965:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1788205284.188967:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000001:3.0:1788205284.188969:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.188976:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205284.188980:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1788205284.188981:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188987:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788205284.188987:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205284.188990:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:1.0:1788205284.188994:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000100:00100000:3.0:1788205284.188996:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543533824:192.168.201.118@tcp:250:kworker.0 00000020:00000040:1.0:1788205284.188997:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:1.0:1788205284.189003:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205284.189004:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788205284.189006:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205284.189009:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205284.189011:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205284.189012:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205284.189016:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205284.189017:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.189020:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 02000000:00000001:3.0:1788205284.189023:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1788205284.189024:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1788205284.189027:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205284.189029:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205284.189031:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 00000100:00000010:1.0:1788205284.189033:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7aadb7e680. 02000000:00000001:3.0:1788205284.189034:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1788205284.189036:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:1.0:1788205284.189040:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788205284.189042:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205284.189045:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205284.189046:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788205284.189048:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788205284.189051:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1788205284.189055:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7aab9bb800. 00000400:00000010:3.0:1788205284.189057:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000100:00000001:1.0:1788205284.189059:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205284.189066:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:3.0:1788205284.189069:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000001:1.0:1788205284.189078:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788205284.189081:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1788205284.189082:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543533824, portal 25 00000100:00000001:3.0:1788205284.189086:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1788205284.189087:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d255ef17] to pc [ptlrpcd_rcv+-1] req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189108:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447291008 : -109424262260608 : ffff9c7aadb7d880) 00000100:00000040:3.0:1788205284.189116:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189133:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788205284.189139:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205284.189139:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205284.189145:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000040:1.0:1788205284.189146:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26980) 00000100:00000200:3.0:1788205284.189151:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543533824, offset 0 00000400:00000200:3.0:1788205284.189160:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205284.189171:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.189178:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205284.189184:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205284.189188:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205284.189192:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205284.189196:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205284.189201:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471519:471519:256:4294967295] 192.168.201.118@tcp LPNI seq info [471378:471378:8:4294967295] 00000400:00000200:3.0:1788205284.189219:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.189229:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205284.189235:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.189241:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205284.189253:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205284.189258:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205284.189263:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205284.189267:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205284.189272:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.189275:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205284.189281:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205284.189286:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205284.189292:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205284.189295:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205284.189299:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205284.189303:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189312:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.189316:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189334:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.189336:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205284.189352:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205284.189357:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205284.189360:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321644294 00000100:00000010:3.0:1788205284.189366:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fc5a0. 00000100:00000001:3.0:1788205284.189373:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205284.189379:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321644294 00000100:00000010:3.0:1788205284.189383:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcde0. 00000100:00000001:3.0:1788205284.189386:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205284.189390:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205284.189392:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.189395:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205284.189400:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205284/real 1788205284] req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205284.189410:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205284.189412:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205284.189414:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205284.189417:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205284.189419:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205284.189422:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205284.189436:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205284.189445:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189451:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205284.189453:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.189456:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.189458:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205284.189462:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189469:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205284.189473:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189485:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205284.189488:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.189490:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.189492:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.189502:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205284.189505:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205284.189508:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205284.189513:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205284.189515:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205284.189519:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205284.189528:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543533824:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205284.189533:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205284.189536:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189543:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205284.189544:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.189547:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205284.189549:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205284.189552:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205284.189555:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205284.189558:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205284.189561:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205284.189562:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.189565:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205284.189568:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205284.189570:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205284.189574:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205284.189577:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205284.189581:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205284.189583:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543533952, portal 10 00000100:00000001:3.0:1788205284.189585:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205284.189587:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447298176 : -109424262253440 : ffff9c7aadb7f480) 00000100:00000040:3.0:1788205284.189593:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189599:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205284.189601:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205284.189603:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205284.189606:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543533952, offset 0 00000400:00000200:3.0:1788205284.189609:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205284.189613:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.189617:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205284.189620:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205284.189622:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205284.189625:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205284.189627:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205284.189630:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471520:471520:256:4294967295] 192.168.201.118@tcp LPNI seq info [471379:471379:8:4294967295] 00000400:00000200:3.0:1788205284.189634:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.189640:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205284.189644:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.189647:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205284.189652:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205284.189654:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205284.189659:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205284.189661:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205284.189665:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.189666:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205284.189669:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205284.189672:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205284.189675:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205284.189678:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205284.189681:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205284.189684:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189691:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.189694:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189700:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.189702:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205284.189705:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205284.189710:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205284.189713:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205284.189715:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321643027 00000100:00000001:3.0:1788205284.189718:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321644294 00000100:00000001:3.0:1788205284.189721:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321643027 00000100:00000001:3.0:1788205284.189723:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321644294 00000100:00000001:3.0:1788205284.189725:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205284.189726:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.189728:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205284.189732:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205284/real 1788205284] req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205284.189740:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205284.189741:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205284.189743:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205284.189745:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205284.189748:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205284.189750:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205284.189753:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205284.189760:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189772:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205284.189774:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.189775:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.189777:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205284.189780:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189787:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205284.189791:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189798:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205284.189801:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.189802:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.189804:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.189806:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205284.189808:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205284.189810:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205284.189813:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205284.189816:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205284.189819:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205284.189827:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543533952:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205284.189831:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205284.189834:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189841:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205284.189843:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.189845:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205284.189847:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205284.189849:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205284.189852:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205284.189855:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205284.189857:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205284.189859:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.189861:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205284.189864:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205284.189866:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205284.189882:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205284.189885:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205284.189888:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205284.189891:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543534080, portal 10 00000100:00000001:3.0:1788205284.189894:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205284.189895:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447286528 : -109424262265088 : ffff9c7aadb7c700) 00000100:00000040:3.0:1788205284.189900:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189907:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205284.189909:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205284.189911:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205284.189913:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543534080, offset 0 00000400:00000200:3.0:1788205284.189917:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205284.189921:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.189924:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205284.189927:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205284.189930:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205284.189932:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205284.189935:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205284.189938:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471521:471521:256:4294967295] 192.168.201.118@tcp LPNI seq info [471380:471380:8:4294967295] 00000400:00000200:3.0:1788205284.189942:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.189948:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205284.189953:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.189957:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205284.189961:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205284.189964:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205284.189968:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205284.189970:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205284.189974:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.189976:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205284.189979:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205284.189982:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205284.189986:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205284.189987:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205284.189990:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205284.189993:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.189999:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.190002:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190007:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.190009:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205284.190012:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205284.190015:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205284.190018:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205284.190020:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321643028 00000100:00000001:3.0:1788205284.190023:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321644294 00000100:00000001:3.0:1788205284.190025:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321643028 00000100:00000001:3.0:1788205284.190027:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321644294 00000100:00000001:3.0:1788205284.190029:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205284.190031:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.190033:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205284.190036:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205284/real 1788205284] req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205284.190045:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205284.190046:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205284.190048:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205284.190051:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205284.190052:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205284.190054:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205284.190057:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205284.190065:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190071:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205284.190072:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190074:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.190075:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205284.190078:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190085:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205284.190107:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190114:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205284.190116:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190118:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.190119:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190121:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205284.190124:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205284.190126:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205284.190129:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205284.190132:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205284.190136:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205284.190143:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534080:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205284.190147:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205284.190150:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190156:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205284.190157:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.190160:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205284.190162:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205284.190164:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205284.190166:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205284.190169:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205284.190171:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205284.190172:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.190174:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205284.190177:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205284.190180:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205284.190184:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205284.190186:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205284.190189:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205284.190192:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543534208, portal 4 00000100:00000001:3.0:1788205284.190194:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205284.190195:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447289216 : -109424262262400 : ffff9c7aadb7d180) 00000100:00000040:3.0:1788205284.190200:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190206:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205284.190208:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205284.190210:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205284.190213:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543534208, offset 0 00000400:00000200:3.0:1788205284.190217:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205284.190221:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.190224:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205284.190227:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205284.190229:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205284.190232:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205284.190234:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205284.190237:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471522:471522:256:4294967295] 192.168.201.118@tcp LPNI seq info [471381:471381:8:4294967295] 00000400:00000200:3.0:1788205284.190241:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.190247:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205284.190251:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.190254:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205284.190258:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205284.190260:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205284.190264:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205284.190267:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205284.190270:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.190272:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205284.190275:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205284.190278:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205284.190281:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205284.190283:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205284.190285:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205284.190289:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190296:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.190298:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190304:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.190306:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205284.190309:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205284.190311:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205284.190315:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205284.190317:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321643038 00000100:00000001:3.0:1788205284.190319:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321644294 00000100:00000001:3.0:1788205284.190322:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321643038 00000100:00000001:3.0:1788205284.190323:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321644294 00000100:00000001:3.0:1788205284.190325:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205284.190327:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.190329:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205284.190333:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205284/real 1788205284] req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205284.190340:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205284.190342:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205284.190343:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205284.190346:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205284.190347:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205284.190349:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205284.190352:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205284.190359:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190365:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205284.190367:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190368:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.190369:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205284.190373:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190379:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205284.190382:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190389:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205284.190392:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190393:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.190395:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190396:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205284.190398:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205284.190401:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205284.190404:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205284.190407:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205284.190410:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205284.190417:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534208:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205284.190421:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190423:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.190426:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d880 x1875055543533824/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190433:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205284.190436:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205284.190439:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205284.190442:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205284.190444:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205284.190446:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205284.190448:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205284.190451:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bbc00. 02000000:00000001:3.0:1788205284.190455:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205284.190457:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205284.190459:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d880. 00000100:00000001:3.0:1788205284.190465:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205284.190466:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190468:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.190471:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7f480 x1875055543533952/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190485:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205284.190486:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205284.190488:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205284.190490:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205284.190491:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205284.190493:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205284.190495:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205284.190496:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba400. 02000000:00000001:3.0:1788205284.190498:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205284.190499:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205284.190501:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7f480. 00000100:00000001:3.0:1788205284.190506:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205284.190508:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190509:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.190512:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7c700 x1875055543534080/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190518:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205284.190519:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205284.190521:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205284.190523:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205284.190524:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205284.190525:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205284.190527:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205284.190529:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9ba000. 02000000:00000001:3.0:1788205284.190530:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205284.190531:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205284.190533:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7c700. 00000100:00000001:3.0:1788205284.190535:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205284.190536:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190538:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.190541:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7d180 x1875055543534208/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190547:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205284.190548:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205284.190550:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205284.190552:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205284.190553:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205284.190554:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205284.190556:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205284.190557:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9b9000. 02000000:00000001:3.0:1788205284.190559:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205284.190561:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205284.190562:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7d180. 00000100:00000001:3.0:1788205284.190564:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205284.190566:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190568:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190576:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205284.190578:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.190580:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205284.190582:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205284.190584:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205284.190587:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190594:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205284.190596:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.190598:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205284.190599:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205284.190601:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205284.190604:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205284.190607:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205284.190609:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205284.190610:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205284.190612:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205284.190615:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205284.190617:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205284.190620:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205284.190622:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205284.190626:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205284.190630:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543534336, portal 4 00000100:00000001:3.0:1788205284.190632:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205284.190634:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634649447294592 : -109424262257024 : ffff9c7aadb7e680) 00000100:00000040:3.0:1788205284.190639:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190645:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205284.190647:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205284.190650:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205284.190652:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543534336, offset 0 00000400:00000200:3.0:1788205284.190656:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205284.190660:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.190664:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205284.190667:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205284.190669:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205284.190671:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205284.190674:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205284.190677:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471523:471523:256:4294967295] 192.168.201.118@tcp LPNI seq info [471382:471382:8:4294967295] 00000400:00000200:3.0:1788205284.190680:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205284.190686:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205284.190690:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.190694:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205284.190697:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205284.190700:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000200:3.1:1788205284.190704:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205284.190707:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 17 secs later) 00000800:00000100:3.0:1788205284.190710:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205284.190712:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205284.190715:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205284.190719:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205284.190722:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205284.190723:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205284.190726:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205284.190730:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190744:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.190747:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205309 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190754:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.190756:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205284.190759:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205284.190762:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205284.190765:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205284.190767:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321643061 00000100:00000001:3.0:1788205284.190769:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321644294 00000100:00000001:3.0:1788205284.190771:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321643061 00000100:00000001:3.0:1788205284.190774:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321644294 00000100:00000001:3.0:1788205284.190776:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205284.190778:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.190780:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205284.190783:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205284/real 1788205284] req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205284.190791:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205284.190792:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205284.190794:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205284.190797:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205284.190798:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205284.190801:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205284.190804:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205284.190812:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190818:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205284.190820:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190821:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.190823:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205284.190826:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190832:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205284.190836:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190842:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205284.190844:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190846:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205284.190847:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190849:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205284.190850:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205284.190853:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205284.190856:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205284.190858:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205284.190862:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205284.190884:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7aadb7e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534336:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205284.190887:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190890:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205284.190892:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7aadb7e680 x1875055543534336/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205309 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205284.190898:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205284.190900:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205284.190902:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205284.190904:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205284.190906:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205284.190907:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205284.190909:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205284.190911:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7aab9bb800. 02000000:00000001:3.0:1788205284.190913:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205284.190914:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205284.190915:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7aadb7e680. 00000100:00000001:3.0:1788205284.190921:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205284.190923:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190925:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205284.190928:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205284.190929:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205284.190931:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205284.190933:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205285.020239:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205285.020251:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205285.020253:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020257:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205285.020261:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205285.020263:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205285.020269:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205285.020272:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205285.020274:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205285.020274:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205285.020277:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205285.020280:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205285.020282:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205285.020285:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205285.020286:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205285.020291:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205285.020293:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00000100:00000001:2.0:1788205285.020294:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020297:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205285.020299:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205285.020303:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205285.020303:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205285.020304:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205285.020310:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205285.020310:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205285.020317:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205285.020318:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205285.020321:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205285.020322:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205285.020325:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:2.0:1788205285.020326:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020328:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020331:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205285.020333:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020335:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020337:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020341:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205285.020344:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205285.020347:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205285.020349:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020351:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205285.020352:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020354:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205285.020356:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205285.020428:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205285.020433:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205285.020436:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1788205285.532245:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321644294, at 4321644352 00000100:00000010:3.0:1788205285.532268:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcde0. 00000100:00000040:2.0:1788205285.532276:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321644294, at 4321644352 00000100:00000010:2.0:1788205285.532286:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fc5a0. 00000100:00000001:3.0:1788205286.044235:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205286.044239:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205286.044248:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205286.044250:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205286.044253:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205286.044258:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00100000:3.0:1788205286.044258:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205286.044261:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205286.044264:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205286.044266:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205286.044269:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205286.044269:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205286.044274:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205286.044275:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205286.044277:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205286.044281:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205286.044287:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205286.044290:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205286.044296:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205286.044298:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205286.044301:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205286.044304:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205286.044306:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205286.044309:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205286.044311:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205286.044313:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205286.044383:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205286.044387:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205286.044390:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205287.068213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205287.068221:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205287.068223:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205287.068228:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205287.068233:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205287.068236:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205287.068239:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205287.068241:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205287.068244:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205287.069934:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205287.069939:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205287.069944:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205287.069945:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205287.069948:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205287.069950:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205287.069953:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205287.069954:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205287.070027:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205287.070031:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205287.070034:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205287.070334:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205287.070337:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205287.070339:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205287.070343:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205287.070347:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205287.070349:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.070352:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205287.070355:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205287.070357:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205288.092213:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205288.092216:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205288.092223:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205288.092223:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205288.092225:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205288.092225:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205288.092231:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205288.092231:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788205288.092236:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205288.092236:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205288.092239:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205288.092239:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205288.092242:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205288.092243:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205288.092245:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205288.092246:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205288.092249:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205288.092249:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205288.092427:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205288.092434:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205288.092442:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205288.092444:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205288.092447:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205288.092448:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205288.092452:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205288.092453:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205288.093053:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205288.093057:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205288.093060:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788205289.116206:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205289.116215:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205289.116217:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205289.116302:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205289.116309:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205289.116317:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205289.116318:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205289.116322:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205289.116323:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205289.116327:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205289.116328:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205289.116446:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205289.116451:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205289.116454:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788205289.116806:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205289.116811:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205289.116815:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205289.116818:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205289.116821:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205289.116824:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.116898:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205289.116901:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205289.116901:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205289.116903:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205289.116904:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205289.116906:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.116907:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205289.116909:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.116911:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205289.309177:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205289.309193:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788205289.309197:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205289.309212:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205289.309217:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205289.309227:0:399478:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26975 00000400:00000200:0.0:1788205289.309240:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205289.309249:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205289.309253:0:399478:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205289.309256:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309260:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309263:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309266:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205289.309271:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205289.309273:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:0.0:1788205289.309277:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205289.309281:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:0.0:1788205289.309284:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:0.0:1788205289.309286:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:0.0:1788205289.309290:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309292:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309294:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309296:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205289.309299:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:0.0:1788205289.309303:0:399478:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309306:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205289.309309:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205289.309314:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205289.309317:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309320:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:0.0:1788205289.309323:0:399478:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205289.309335:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8000. 00000020:00000040:0.0:1788205289.309339:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:0.0:1788205289.309344:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205289.309345:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309350:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205289.309353:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205289.309359:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205289.309368:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28400. 00000100:00000001:0.0:1788205289.309377:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205289.309387:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205289.309399:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205289.309404:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205289.309409:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c192f41f] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205289.309445:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205289.309448:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205289.309452:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788205289.309454:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205289.309455:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205289.309458:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205289.309461:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26975 00000400:00000200:0.0:1788205289.309465:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205289.309469:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205289.309471:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205289.309474:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309476:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309478:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309480:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205289.309483:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205289.309485:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:0.0:1788205289.309487:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205289.309489:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:0.0:1788205289.309491:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:0.0:1788205289.309493:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:0.0:1788205289.309496:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309498:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309500:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309502:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205289.309505:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:0.0:1788205289.309507:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309510:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205289.309512:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205289.309519:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205289.309521:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309522:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205289.309527:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205289.309530:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309532:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205289.309536:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75db100. 00000020:00000040:0.0:1788205289.309538:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205289.309541:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205289.309542:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309544:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205289.309545:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205289.309547:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205289.309551:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28c00. 00000100:00000001:0.0:1788205289.309554:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205289.309559:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205289.309566:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205289.309568:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205289.309572:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019ee8a8b] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205289.309580:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205289.309583:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205289.309586:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1788205289.309587:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205289.309589:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205289.309591:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205289.309594:0:399478:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26975 00000400:00000200:0.0:1788205289.309597:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205289.309610:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205289.309613:0:399478:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205289.309615:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309617:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309619:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309621:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205289.309623:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205289.309624:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:0.0:1788205289.309626:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205289.309628:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:0.0:1788205289.309630:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:0.0:1788205289.309632:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:0.0:1788205289.309635:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309636:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309638:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309640:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205289.309642:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:0.0:1788205289.309645:0:399478:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309647:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205289.309649:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205289.309652:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205289.309653:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309655:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205289.309657:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788205289.309659:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309661:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205289.309665:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8700. 00000020:00000040:0.0:1788205289.309667:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:0.0:1788205289.309669:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205289.309670:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309672:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205289.309674:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205289.309675:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205289.309680:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2ac00. 00000100:00000001:0.0:1788205289.309683:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205289.309688:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205289.309695:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205289.309697:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205289.309701:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000324022a6] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205289.309708:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205289.309711:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205289.309714:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788205289.309715:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205289.309717:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205289.309720:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205289.309723:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26975 00000400:00000200:0.0:1788205289.309726:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205289.309729:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205289.309731:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205289.309734:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309736:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309738:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309740:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205289.309742:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205289.309743:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:0.0:1788205289.309744:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205289.309746:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:0.0:1788205289.309748:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:0.0:1788205289.309751:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:0.0:1788205289.309754:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309755:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309757:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309758:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205289.309760:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:0.0:1788205289.309763:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309765:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205289.309767:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205289.309769:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205289.309772:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309774:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205289.309776:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:0.0:1788205289.309778:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309812:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205289.309815:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75d8380. 00000020:00000040:0.0:1788205289.309818:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205289.309820:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205289.309821:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309823:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205289.309824:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205289.309826:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205289.309829:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c2a800. 00000100:00000001:0.0:1788205289.309832:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205289.309837:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205289.309843:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205289.309845:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205289.309848:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000035c75eaf] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205289.309854:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788205289.309857:0:399478:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788205289.309860:0:399478:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788205289.309861:0:399478:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788205289.309862:0:399478:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788205289.309866:0:399478:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1788205289.309868:0:399478:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26975 00000400:00000200:0.0:1788205289.309871:0:399478:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:0.0:1788205289.309874:0:399478:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:0.0:1788205289.309876:0:399478:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1788205289.309878:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309880:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309882:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309883:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:0.0:1788205289.309885:0:399478:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:0.0:1788205289.309886:0:399478:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:0.0:1788205289.309888:0:399478:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:0.0:1788205289.309890:0:399478:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:0.0:1788205289.309891:0:399478:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:0.0:1788205289.309893:0:399478:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:0.0:1788205289.309896:0:399478:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309897:0:399478:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1788205289.309900:0:399478:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309902:0:399478:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:0.0:1788205289.309904:0:399478:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:0.0:1788205289.309907:0:399478:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:0.0:1788205289.309910:0:399478:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788205289.309912:0:399478:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788205289.309914:0:399478:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788205289.309916:0:399478:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309918:0:399478:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:0.0:1788205289.309920:0:399478:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1788205289.309922:0:399478:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309924:0:399478:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1788205289.309928:0:399478:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7bb75dad80. 00000020:00000040:0.0:1788205289.309930:0:399478:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:0.0:1788205289.309932:0:399478:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788205289.309933:0:399478:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788205289.309935:0:399478:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788205289.309937:0:399478:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788205289.309939:0:399478:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1788205289.309943:0:399478:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7b88c28000. 00000100:00000001:0.0:1788205289.309946:0:399478:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788205289.309951:0:399478:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788205289.309958:0:399478:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788205289.309960:0:399478:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205289.309963:0:399478:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003cb38392] to pc [ptlrpcd_rcv+-1] req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788205289.309972:0:399478:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1788205289.309979:0:399478:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26985) 00000100:00000001:3.0:1788205289.311217:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205289.311226:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205289.311230:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205289.311240:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311250:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205289.311253:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.311257:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205289.311261:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205289.311264:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205289.311268:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543534464:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205289.311273:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205289.311279:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205289.311282:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.311285:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205289.311293:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205289.311295:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205289.311302:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205289.311307:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205289.311317:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205289.311326:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543534464, portal 25 00000100:00000001:3.0:1788205289.311328:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205289.311330:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904109568 : -109419805442048 : ffff9c7bb75d8000) 00000100:00000040:3.0:1788205289.311337:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311345:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205289.311347:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205289.311350:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205289.311353:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543534464, offset 0 00000400:00000200:3.0:1788205289.311360:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205289.311371:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.311379:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205289.311385:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205289.311390:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205289.311394:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205289.311397:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205289.311402:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471524:471524:256:4294967295] 192.168.201.118@tcp LPNI seq info [471383:471383:8:4294967295] 00000400:00000200:3.0:1788205289.311406:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.311417:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205289.311424:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.311431:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205289.311439:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205289.311443:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205289.311449:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205289.311454:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205289.311459:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.311463:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205289.311470:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205289.311474:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205289.311481:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205289.311484:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205289.311488:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205289.311493:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311504:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.311508:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311515:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.311517:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205289.311524:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205289.311529:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205289.311532:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321649416 00000100:00000010:3.0:1788205289.311538:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcde0. 00000100:00000001:3.0:1788205289.311546:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1788205289.311554:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321649416 00000100:00000010:3.0:1788205289.311557:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab06fcba0. 00000100:00000001:3.0:1788205289.311561:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1788205289.311564:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205289.311567:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.311570:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205289.311574:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205289/real 1788205289] req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205289.311585:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205289.311587:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205289.311589:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205289.311592:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205289.311594:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205289.311597:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205289.311601:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205289.311609:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311615:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205289.311617:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.311620:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.311622:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205289.311626:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311632:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205289.311638:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311646:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205289.311649:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.311651:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.311653:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.311656:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205289.311658:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205289.311662:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205289.311667:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205289.311670:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205289.311674:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205289.311683:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543534464:192.168.201.118@tcp:250:kworker.0 00000100:00000001:3.0:1788205289.311689:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205289.311692:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311716:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205289.311717:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.311720:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205289.311722:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205289.311724:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205289.311727:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534592:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205289.311730:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205289.311733:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205289.311735:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.311736:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205289.311740:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205289.311742:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205289.311746:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205289.311748:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205289.311753:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205289.311756:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543534592, portal 10 00000100:00000001:3.0:1788205289.311758:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205289.311759:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904122112 : -109419805429504 : ffff9c7bb75db100) 00000100:00000040:3.0:1788205289.311766:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311772:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205289.311775:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205289.311777:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205289.311789:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543534592, offset 0 00000400:00000200:3.0:1788205289.311794:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205289.311799:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.311802:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205289.311805:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205289.311807:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205289.311809:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205289.311812:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205289.311815:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471525:471525:256:4294967295] 192.168.201.118@tcp LPNI seq info [471384:471384:8:4294967295] 00000400:00000200:3.0:1788205289.311818:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.311824:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205289.311829:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.311833:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205289.311836:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205289.311839:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205289.311843:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205289.311846:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205289.311850:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.311852:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205289.311855:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205289.311858:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205289.311861:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205289.311863:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205289.311865:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205289.311869:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311877:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.311880:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311886:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.311888:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205289.311891:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205289.311895:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:3.0:1788205289.311899:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205289.311901:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321648149 00000100:00000001:3.0:1788205289.311904:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321649416 00000100:00000001:3.0:1788205289.311906:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321648149 00000100:00000001:3.0:1788205289.311908:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321649416 00000100:00000001:3.0:1788205289.311910:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205289.311912:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.311914:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205289.311918:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205289/real 1788205289] req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205289.311926:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205289.311927:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205289.311929:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205289.311932:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205289.311934:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205289.311936:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205289.311939:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205289.311947:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311953:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205289.311954:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.311956:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.311957:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205289.311961:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311967:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205289.311971:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.311978:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205289.311980:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.311982:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.311983:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.311985:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205289.311989:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205289.311991:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205289.311995:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205289.311997:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205289.312000:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205289.312008:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75db100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534592:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205289.312012:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205289.312017:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312024:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205289.312026:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.312028:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205289.312030:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205289.312032:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205289.312036:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534720:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205289.312039:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205289.312042:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205289.312043:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.312045:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205289.312049:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205289.312051:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205289.312055:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205289.312057:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205289.312060:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205289.312063:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543534720, portal 10 00000100:00000001:3.0:1788205289.312065:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205289.312067:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904111360 : -109419805440256 : ffff9c7bb75d8700) 00000100:00000040:3.0:1788205289.312087:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312161:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205289.312164:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205289.312168:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205289.312171:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543534720, offset 0 00000400:00000200:3.0:1788205289.312174:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205289.312179:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.312183:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205289.312185:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205289.312188:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205289.312190:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205289.312192:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205289.312196:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471526:471526:256:4294967295] 192.168.201.118@tcp LPNI seq info [471385:471385:8:4294967295] 00000400:00000200:3.0:1788205289.312200:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.312206:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205289.312211:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.312214:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205289.312218:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205289.312221:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205289.312225:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205289.312228:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205289.312232:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.312234:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205289.312237:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205289.312241:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205289.312244:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205289.312246:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205289.312248:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205289.312252:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312260:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.312264:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312270:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.312272:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205289.312275:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205289.312278:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:3.0:1788205289.312281:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205289.312283:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321648151 00000100:00000001:3.0:1788205289.312286:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321649416 00000100:00000001:3.0:1788205289.312288:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321648151 00000100:00000001:3.0:1788205289.312290:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321649416 00000100:00000001:3.0:1788205289.312292:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205289.312294:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.312296:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205289.312299:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205289/real 1788205289] req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205289.312308:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205289.312309:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205289.312311:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205289.312314:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205289.312315:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205289.312317:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205289.312320:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205289.312328:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312334:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205289.312335:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312337:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.312338:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205289.312342:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312349:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205289.312353:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312360:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205289.312362:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312364:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.312365:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312367:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205289.312371:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205289.312374:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205289.312378:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205289.312380:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205289.312384:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205289.312394:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534720:192.168.201.118@tcp:38:kworker.0 00000100:00000001:3.0:1788205289.312398:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205289.312402:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312409:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205289.312411:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.312414:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205289.312416:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205289.312418:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205289.312420:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534848:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205289.312424:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205289.312427:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205289.312429:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.312430:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205289.312434:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205289.312436:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205289.312441:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205289.312443:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205289.312446:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205289.312449:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543534848, portal 4 00000100:00000001:3.0:1788205289.312451:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205289.312453:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904110464 : -109419805441152 : ffff9c7bb75d8380) 00000100:00000040:3.0:1788205289.312458:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312465:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205289.312467:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205289.312470:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205289.312472:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543534848, offset 0 00000400:00000200:3.0:1788205289.312476:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205289.312480:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.312484:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205289.312487:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205289.312489:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205289.312492:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205289.312495:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205289.312497:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471527:471527:256:4294967295] 192.168.201.118@tcp LPNI seq info [471386:471386:8:4294967295] 00000400:00000200:3.0:1788205289.312503:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.312509:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205289.312513:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.312516:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205289.312520:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205289.312523:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205289.312527:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205289.312530:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205289.312533:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.312535:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205289.312538:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205289.312541:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205289.312544:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205289.312546:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205289.312549:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205289.312552:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312559:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.312561:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312567:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.312568:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205289.312571:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205289.312574:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:3.0:1788205289.312578:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205289.312580:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321648160 00000100:00000001:3.0:1788205289.312582:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321649416 00000100:00000001:3.0:1788205289.312585:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321648160 00000100:00000001:3.0:1788205289.312587:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321649416 00000100:00000001:3.0:1788205289.312589:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205289.312591:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.312593:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205289.312596:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205289/real 1788205289] req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205289.312604:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205289.312605:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205289.312607:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205289.312610:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205289.312611:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205289.312613:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205289.312616:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205289.312624:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312631:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205289.312632:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312634:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.312635:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205289.312639:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312645:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205289.312648:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312655:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205289.312658:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312659:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.312661:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312663:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205289.312665:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205289.312667:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205289.312671:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205289.312673:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205289.312677:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205289.312686:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75d8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534848:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205289.312690:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1788205289.312694:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312701:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205289.312703:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.312705:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788205289.312707:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205289.312709:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205289.312712:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534976:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205289.312715:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788205289.312717:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788205289.312719:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205289.312721:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1788205289.312724:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788205289.312727:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1788205289.312730:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:3.0:1788205289.312732:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7aacb77840. 00000400:00000010:3.0:1788205289.312736:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734990. 00000100:00000200:3.0:1788205289.312739:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543534976, portal 4 00000100:00000001:3.0:1788205289.312741:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788205289.312742:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653904121216 : -109419805430400 : ffff9c7bb75dad80) 00000100:00000040:3.0:1788205289.312749:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312756:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1788205289.312758:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:3.0:1788205289.312760:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b98734b28. 00000100:00000200:3.0:1788205289.312762:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543534976, offset 0 00000400:00000200:3.0:1788205289.312766:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:3.0:1788205289.312770:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.312773:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:3.0:1788205289.312776:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788205289.312789:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:3.0:1788205289.312792:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788205289.312794:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:3.0:1788205289.312798:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471528:471528:256:4294967295] 192.168.201.118@tcp LPNI seq info [471387:471387:8:4294967295] 00000400:00000200:3.0:1788205289.312802:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205289.312808:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:3.0:1788205289.312813:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.312816:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7ab60bda00. 00000800:00000200:3.0:1788205289.312820:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.0:1788205289.312822:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000200:3.1:1788205289.312827:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:3.1:1788205289.312829:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 12 secs later) 00000800:00000100:3.0:1788205289.312833:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:3.0:1788205289.312835:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7ab60bda00. 00000400:00000200:3.0:1788205289.312838:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:3.0:1788205289.312841:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1788205289.312844:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734b28 00000400:00000010:3.0:1788205289.312846:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734b28. 00000100:00000001:3.0:1788205289.312848:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1788205289.312852:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312859:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.312862:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205314 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312869:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.312870:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1788205289.312873:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205289.312876:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:3.0:1788205289.312879:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:3.0:1788205289.312881:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321648183 00000100:00000001:3.0:1788205289.312884:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321649416 00000100:00000001:3.0:1788205289.312886:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321648183 00000100:00000001:3.0:1788205289.312888:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321649416 00000100:00000001:3.0:1788205289.312890:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788205289.312892:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.312894:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788205289.312897:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205289/real 1788205289] req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788205289.312905:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788205289.312906:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1788205289.312908:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7aacb77840. 00000400:00000200:3.0:1788205289.312910:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b98734990 00000400:00000010:3.0:1788205289.312911:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b98734990. 00000100:00000001:3.0:1788205289.312914:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1788205289.312916:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788205289.312924:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312930:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788205289.312932:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312933:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.312935:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788205289.312938:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312944:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1788205289.312948:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312955:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788205289.312957:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312958:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788205289.312959:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312961:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788205289.312963:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788205289.312965:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788205289.312968:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:3.0:1788205289.312970:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1788205289.312974:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788205289.312980:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7bb75dad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543534976:192.168.201.118@tcp:8:kworker.0 00000100:00000001:3.0:1788205289.312984:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.312986:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.312989:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8000 x1875055543534464/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.312996:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205289.312999:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205289.313002:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:3.0:1788205289.313005:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205289.313007:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205289.313009:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:3.0:1788205289.313012:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205289.313015:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28400. 02000000:00000001:3.0:1788205289.313019:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205289.313021:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205289.313023:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8000. 00000100:00000001:3.0:1788205289.313027:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205289.313028:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.313030:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.313033:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75db100 x1875055543534592/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.313038:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205289.313039:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205289.313041:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:3.0:1788205289.313043:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205289.313044:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205289.313045:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205289.313047:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205289.313048:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28c00. 02000000:00000001:3.0:1788205289.313049:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205289.313050:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205289.313266:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75db100. 00000100:00000001:3.0:1788205289.313277:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205289.313279:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.313284:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.313296:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8700 x1875055543534720/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.313306:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205289.313309:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205289.313313:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:3.0:1788205289.313318:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205289.313320:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205289.313322:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:3.0:1788205289.313324:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205289.313327:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2ac00. 02000000:00000001:3.0:1788205289.313330:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205289.313331:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205289.313333:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8700. 00000100:00000001:3.0:1788205289.313335:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205289.313336:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.313338:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.313343:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75d8380 x1875055543534848/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.313350:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205289.313351:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205289.313352:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:3.0:1788205289.313354:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205289.313355:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205289.313356:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205289.313358:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205289.313360:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c2a800. 02000000:00000001:3.0:1788205289.313361:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205289.313362:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205289.313364:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75d8380. 00000100:00000001:3.0:1788205289.313366:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205289.313367:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.313369:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1788205289.313373:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7bb75dad80 x1875055543534976/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205314 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205289.313379:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788205289.313381:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1788205289.313382:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:3.0:1788205289.313384:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788205289.313385:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:3.0:1788205289.313386:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:3.0:1788205289.313388:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:3.0:1788205289.313389:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7b88c28000. 02000000:00000001:3.0:1788205289.313391:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788205289.313392:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1788205289.313393:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7bb75dad80. 00000100:00000001:3.0:1788205289.313396:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788205289.313397:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.313399:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205289.313409:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205289.313411:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205289.313414:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788205289.313416:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205290.140206:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205290.140215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205290.140217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205290.140224:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00010000:00000001:0.0:1788205290.140232:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:3.0:1788205290.140232:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205290.140237:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205290.140244:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00000100:00000001:3.0:1788205290.140244:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1788205290.140249:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00000100:00000001:3.0:1788205290.140250:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205290.140254:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205290.140256:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205290.140261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205290.140264:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205290.140266:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205290.140269:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205290.140273:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205290.140274:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205290.140277:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205290.140278:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205290.140282:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205290.140283:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205290.141512:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205290.141523:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205290.141526:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205290.141637:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205290.141641:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205290.141642:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205290.141644:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205290.141647:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205290.141650:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205290.141652:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205290.141654:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205290.141656:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205290.652275:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321649416, at 4321649472 00000100:00000010:2.0:1788205290.652294:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcde0. 00000100:00000040:3.0:1788205290.652368:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321649416, at 4321649472 00000100:00000010:3.0:1788205290.652377:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab06fcba0. 00000100:00000001:2.0:1788205291.164214:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205291.164214:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205291.164222:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205291.164223:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205291.164226:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205291.164226:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205291.164234:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205291.164234:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205291.164240:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205291.164240:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205291.164246:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205291.164247:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205291.164252:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205291.164253:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205291.164256:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205291.164257:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205291.164260:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205291.164262:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205291.164734:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205291.164737:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205291.164742:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205291.164743:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205291.164773:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205291.164774:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205291.164778:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205291.164780:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205291.164879:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205291.164883:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205291.164886:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205292.188194:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205292.188194:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205292.188204:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205292.188205:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205292.188207:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205292.188208:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205292.188216:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00100000:3.0:1788205292.188216:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205292.188222:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205292.188222:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205292.188227:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205292.188228:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205292.188231:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205292.188233:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205292.188235:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205292.188237:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205292.188241:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205292.188242:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205292.188507:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205292.188512:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205292.188516:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205292.188517:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205292.188521:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205292.188522:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205292.188526:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205292.188527:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205292.189464:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205292.189467:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205292.189470:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205293.212207:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205293.212215:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205293.212217:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205293.212222:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205293.212226:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205293.212229:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205293.212232:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205293.212234:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205293.212237:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205293.212378:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205293.212382:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205293.212387:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205293.212388:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205293.212392:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205293.212393:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205293.212396:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205293.212398:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205293.212505:0:404865:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205293.212509:0:404865:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205293.212512:0:404865:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205293.212780:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205293.212784:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205293.212785:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205293.212788:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:2.0:1788205293.212791:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205293.212793:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205293.212795:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205293.212798:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205293.212800:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205294.236201:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00010000:00000001:0.0:1788205294.236206:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205294.236209:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1788205294.236212:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205294.236213:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205294.236219:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00010000:00000001:0.0:1788205294.236220:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205294.236223:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205294.236225:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1788205294.236228:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00000100:00000001:2.0:1788205294.236229:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205294.236231:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205294.236233:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205294.236236:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1788205294.236237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205294.236239:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788205294.236240:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788205294.236354:0:416877:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788205294.236358:0:416877:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788205294.236361:0:416877:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788205294.238060:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1788205294.238065:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788205294.238067:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788205294.238070:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:2.0:1788205294.238073:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788205294.238076:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205294.238079:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788205294.238082:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788205294.238084:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205294.428370:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 09e4f795-47ea-4610-8f6a-0df72c3a64b7->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205294.428381:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788205294.428384:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205294.428388:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205294.428393:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205294.428400:0:409030:0:(import.c:510:import_select_connection()) MGC192.168.201.118@tcp: connect to NID 192.168.201.118@tcp last attempt 26980 00000400:00000200:3.0:1788205294.428412:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205294.428419:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205294.428423:0:409030:0:(import.c:589:import_select_connection()) MGC192.168.201.118@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205294.428426:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.428429:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.428432:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.428435:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205294.428450:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205294.428453:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9551d 00000020:00000001:3.0:1788205294.428456:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205294.428460:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7bb7448800 refcount=3 00000020:00000001:3.0:1788205294.428462:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000020:00000001:3.0:1788205294.428464:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653902473216 : -109419807078400 : ffff9c7bb7448800) 00000100:00000040:3.0:1788205294.428467:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.428469:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.428471:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.428473:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205294.428476:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7bb7448800 : new refcount 2 00000100:00080000:3.0:1788205294.428479:0:409030:0:(import.c:624:import_select_connection()) MGC192.168.201.118@tcp: import ffff9c7b82bec800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205294.428482:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205294.428485:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205294.428488:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205294.428490:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.428492:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000100:00000001:3.0:1788205294.428495:0:409030:0:(obd_class.h:896:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205294.428507:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39f80. 00000020:00000040:3.0:1788205294.428510:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b82bec800 refcount=4 obd=MGC192.168.201.118@tcp 02000000:00000001:3.0:1788205294.428513:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205294.428515:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.428519:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205294.428521:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205294.428526:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205294.428533:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7413000. 00000100:00000001:3.0:1788205294.428540:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205294.428548:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205294.428559:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205294.428564:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205294.428569:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000096d358e] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.428635:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205294.428639:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205294.428642:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205294.428648:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.428656:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205294.428658:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.428673:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205294.428675:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205294.428678:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205294.428682:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543535104:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205294.428686:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205294.428701:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205294.428704:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.428706:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205294.428713:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205294.428715:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205294.428722:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205294.428726:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205294.428735:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205294.428744:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543535104, portal 25 00000100:00000001:1.0:1788205294.428746:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205294.428747:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643415424 : -109420066136192 : ffff9c7ba7d39f80) 00000100:00000040:1.0:1788205294.428753:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.428759:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205294.428762:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205294.428764:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205294.428767:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1875055543535104, offset 0 00000400:00000200:1.0:1788205294.428774:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205294.428782:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.428789:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205294.428794:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205294.428798:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205294.428802:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205294.428804:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205294.428808:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471529:471529:256:4294967295] 192.168.201.118@tcp LPNI seq info [471388:471388:8:4294967295] 00000400:00000200:1.0:1788205294.428812:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.428823:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205294.428829:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.428834:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205294.428841:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205294.428844:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205294.428850:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205294.428853:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205294.428857:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.428860:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205294.428867:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205294.428872:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205294.428879:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205294.428881:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205294.428886:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205294.428890:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205294.428974:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205294.428978:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205294.428982:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788205294.428984:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205294.428986:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205294.428989:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205294.428992:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26980 00000400:00000200:3.0:1788205294.428996:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205294.428999:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205294.429002:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205294.429005:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.429007:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.429009:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.429011:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205294.429013:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205294.429014:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95571 00000020:00000001:3.0:1788205294.429017:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205294.429019:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86464800 refcount=5 00000020:00000001:3.0:1788205294.429021:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000020:00000001:3.0:1788205294.429023:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080504320 : -109420629047296 : ffff9c7b86464800) 00000100:00000040:3.0:1788205294.429026:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.429028:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.429029:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.429031:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205294.429033:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86464800 : new refcount 4 00000100:00080000:3.0:1788205294.429036:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9c7bb744a000: import ffff9c7b90e4b800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205294.429038:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205294.429040:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205294.429045:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205294.429047:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.429049:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205294.429053:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205294.429056:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.429058:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205294.429062:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205294.429064:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7b90e4b800 refcount=8 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205294.429067:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205294.429068:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.429070:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205294.429072:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205294.429074:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205294.429077:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7410000. 00000100:00000001:3.0:1788205294.429081:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205294.429086:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205294.429138:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205294.429140:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205294.429145:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429428:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.429434:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429517:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.429519:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205294.429528:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205294.429532:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205294.429535:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 2 is 4321654534 00000100:00000010:1.0:1788205294.429541:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab1647300. 00000100:00000001:1.0:1788205294.429548:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 2 00000100:00000001:1.0:1788205294.429555:0:256132:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS new deadline estimation for cpu 3 is 4321654534 00000100:00000010:1.0:1788205294.429558:0:256132:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c7ab16479c0. 00000100:00000001:1.0:1788205294.429561:0:256132:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.118@tcp: PM QoS request now active for cpu 3 00000100:00000001:1.0:1788205294.429564:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205294.429566:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.429569:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205294.429573:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205294/real 1788205294] req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205294.429590:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205294.429591:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205294.429594:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205294.429596:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205294.429598:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205294.429600:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205294.429603:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205294.429612:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429619:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205294.429621:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.429623:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.429625:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205294.429629:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429637:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205294.429641:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429648:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205294.429651:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.429653:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.429654:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.429656:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205294.429658:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205294.429670:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b82bec800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205294.429675:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205294.429677:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205294.429681:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205294.429697:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:09e4f795-47ea-4610-8f6a-0df72c3a64b7:256132:1875055543535104:192.168.201.118@tcp:250:kworker.0 00000100:00000001:1.0:1788205294.429702:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.429704:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.429707:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39f80 x1875055543535104/t0(0) o250->MGC192.168.201.118@tcp@192.168.201.118@tcp:26/25 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429714:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205294.429717:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205294.429719:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205294.429723:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205294.429724:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205294.429726:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b82bec800 refcount=3 obd=MGC192.168.201.118@tcp 00000020:00000001:1.0:1788205294.429728:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205294.429730:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7413000. 02000000:00000001:1.0:1788205294.429734:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205294.429736:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205294.429737:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39f80. 00000100:00000001:1.0:1788205294.429741:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205294.429742:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.429744:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.429753:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205294.429755:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.429758:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205294.429759:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205294.429760:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205294.429764:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429771:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205294.429772:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.429775:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205294.429777:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205294.429779:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205294.429782:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543535232:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205294.429786:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205294.429788:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205294.429789:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.429791:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205294.429794:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205294.429796:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205294.429800:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205294.429802:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205294.429806:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205294.429809:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543535232, portal 10 00000100:00000001:1.0:1788205294.429811:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205294.429813:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205294.429819:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429826:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205294.429828:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205294.429830:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205294.429833:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543535232, offset 0 00000400:00000200:1.0:1788205294.429837:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205294.429841:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.429844:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205294.429848:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205294.429850:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205294.429853:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205294.429855:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205294.429859:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471530:471530:256:4294967295] 192.168.201.118@tcp LPNI seq info [471389:471389:8:4294967295] 00000400:00000200:1.0:1788205294.429863:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.429869:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205294.429874:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.429877:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205294.429881:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205294.429884:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205294.429889:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205294.429891:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205294.429895:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.429897:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205294.429900:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205294.429904:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205294.429907:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205294.429909:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205294.429911:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205294.429914:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429920:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.429923:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429929:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.429930:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205294.429932:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205294.429936:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: using a timeout of 17377 usecs (18 jiffies) 00000400:00000200:1.0:1788205294.429939:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205294.429940:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321653267 00000100:00000001:1.0:1788205294.429943:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321654534 00000100:00000001:1.0:1788205294.429945:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321653267 00000100:00000001:1.0:1788205294.429947:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321654534 00000100:00000001:1.0:1788205294.429948:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205294.429950:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.429952:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205294.429954:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205294/real 1788205294] req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205294.429961:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205294.429962:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205294.429964:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205294.429966:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205294.429967:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205294.429969:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205294.429972:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205294.429980:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429986:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205294.429987:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.429988:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.429989:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205294.429993:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.429999:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205294.430002:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.430008:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205294.430010:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.430011:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.430013:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.430014:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205294.430016:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205294.430019:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7b90e4b800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205294.430022:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205294.430024:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205294.430027:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205294.430035:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543535232:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205294.430039:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.430041:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.430043:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543535232/t0(0) o38->lustre-MDT0000-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.430049:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205294.430051:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205294.430052:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205294.430054:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205294.430055:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205294.430057:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7b90e4b800 refcount=7 obd=lustre-MDT0000-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205294.430058:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205294.430060:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7410000. 02000000:00000001:1.0:1788205294.430062:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205294.430063:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205294.430064:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205294.430067:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205294.430069:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.430070:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.430073:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205294.430074:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.430076:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205294.430077:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430317:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205294.430321:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205294.430324:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1788205294.430325:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205294.430327:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205294.430330:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205294.430333:0:409030:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26980 00000400:00000200:3.0:1788205294.430337:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205294.430340:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205294.430343:0:409030:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205294.430347:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430349:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.430351:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430353:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205294.430355:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205294.430356:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95578 00000020:00000001:3.0:1788205294.430358:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205294.430360:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7b86467800 refcount=5 00000020:00000001:3.0:1788205294.430361:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000020:00000001:3.0:1788205294.430363:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634653080516608 : -109420629035008 : ffff9c7b86467800) 00000100:00000040:3.0:1788205294.430366:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430368:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.430369:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430371:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205294.430374:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7b86467800 : new refcount 4 00000100:00080000:3.0:1788205294.430377:0:409030:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff9c7bb744a000: import ffff9c7a91809800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430379:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205294.430380:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205294.430383:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205294.430385:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430387:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205294.430389:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34682651 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788205294.430390:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430392:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205294.430395:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d39c00. 00000020:00000040:3.0:1788205294.430397:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7a91809800 refcount=28 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 02000000:00000001:3.0:1788205294.430400:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205294.430401:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430403:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205294.430405:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205294.430407:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205294.430411:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412c00. 00000100:00000001:3.0:1788205294.430415:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205294.430419:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205294.430426:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205294.430428:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205294.430432:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000048a67498] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.430515:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205294.430517:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.430564:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205294.430566:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205294.430568:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205294.430573:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.430580:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205294.430581:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.430584:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205294.430586:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205294.430588:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205294.430591:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543535360:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205294.430595:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205294.430597:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205294.430598:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.430600:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205294.430603:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205294.430605:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205294.430609:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205294.430611:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205294.430615:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205294.430618:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543535360, portal 10 00000100:00000001:1.0:1788205294.430621:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205294.430622:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643414528 : -109420066137088 : ffff9c7ba7d39c00) 00000100:00000040:1.0:1788205294.430628:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.430634:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205294.430636:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205294.430638:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205294.430641:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1875055543535360, offset 0 00000400:00000200:1.0:1788205294.430644:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205294.430649:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.430652:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205294.430655:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205294.430658:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000100:00000001:3.0:1788205294.430725:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205294.430729:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205294.430733:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788205294.430734:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205294.430736:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205294.430739:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205294.430742:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26980 00000400:00000200:3.0:1788205294.430746:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205294.430749:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205294.430752:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205294.430754:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430756:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.430758:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430760:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205294.430762:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205294.430763:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a95586 00000020:00000001:3.0:1788205294.430765:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205294.430767:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180c000 refcount=4 00000020:00000001:3.0:1788205294.430769:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000020:00000001:3.0:1788205294.430771:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973918208 : -109424735633408 : ffff9c7a9180c000) 00000100:00000040:3.0:1788205294.430774:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430775:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.430778:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430780:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205294.430782:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180c000 : new refcount 3 00000100:00080000:3.0:1788205294.430785:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9c7bb744a000: import ffff9c7aa3cf8800 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430787:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205294.430788:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205294.430791:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205294.430793:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430795:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205294.430797:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 28672, lost: 0. 00000008:00000001:3.0:1788205294.430799:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430801:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205294.430805:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d3b480. 00000020:00000040:3.0:1788205294.430807:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cf8800 refcount=7 obd=lustre-OST0000-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205294.430809:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205294.430810:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430812:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205294.430814:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205294.430815:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205294.430819:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7411c00. 00000100:00000001:3.0:1788205294.430822:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205294.430826:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205294.430834:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205294.430836:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205294.430840:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000011d5ced2] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205294.430847:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788205294.430850:0:409030:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2d12ff60-d499-4808-9599-5b7284f3b7c2->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788205294.430853:0:409030:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788205294.430854:0:409030:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788205294.430856:0:409030:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788205294.430859:0:409030:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1788205294.430861:0:409030:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: connect to NID 192.168.201.118@tcp last attempt 26980 00000400:00000200:3.0:1788205294.430864:0:409030:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:3.0:1788205294.430866:0:409030:0:(peer.c:1803:LNetPeerDiscovered()) Peer NID 192.168.201.118@tcp is discovered: rc = 1 00000100:00080000:3.0:1788205294.430869:0:409030:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1788205294.430871:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430873:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.430875:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430877:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000001:3.0:1788205294.430878:0:409030:0:(genops.c:814:class_conn2export()) Process entered 00000020:00000040:3.0:1788205294.430880:0:409030:0:(genops.c:826:class_conn2export()) looking for export cookie 0x692d9b7161a9558d 00000020:00000001:3.0:1788205294.430881:0:409030:0:(lustre_handles.c:129:class_handle2object()) Process entered 00000020:00000040:3.0:1788205294.430883:0:409030:0:(lustre_handles.c:147:class_handle2object()) GET export ffff9c7a9180f800 refcount=4 00000020:00000001:3.0:1788205294.430885:0:409030:0:(lustre_handles.c:154:class_handle2object()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000020:00000001:3.0:1788205294.430887:0:409030:0:(genops.c:828:class_conn2export()) Process leaving (rc=18446634648973932544 : -109424735619072 : ffff9c7a9180f800) 00000100:00000040:3.0:1788205294.430890:0:409030:0:(lustre_net.h:1975:ptlrpc_connection_put()) PUT conn=ffff9c7b91386440 refcount 14 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430891:0:409030:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1788205294.430893:0:409030:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c7b91386440 refcount 15 to 192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430895:0:409030:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634653264143424 : -109420445408192 : ffff9c7b91386440) 00000020:00000040:3.0:1788205294.430897:0:409030:0:(genops.c:901:class_export_put()) PUTting export ffff9c7a9180f800 : new refcount 3 00000100:00080000:3.0:1788205294.430899:0:409030:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9c7bb744a000: import ffff9c7aa3cfd000 using connection 192.168.201.118@tcp/192.168.201.118@tcp 00000100:00000001:3.0:1788205294.430901:0:409030:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788205294.430902:0:409030:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788205294.430904:0:409030:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788205294.430906:0:409030:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430908:0:409030:0:(obd_class.h:889:obd_reconnect()) Process entered 00000008:00100000:3.0:1788205294.430909:0:409030:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34682651 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1788205294.430911:0:409030:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430913:0:409030:0:(obd_class.h:903:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1788205294.430918:0:409030:0:(client.c:593:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c7ba7d38000. 00000020:00000040:3.0:1788205294.430920:0:409030:0:(genops.c:1232:class_import_get()) import ffff9c7aa3cfd000 refcount=6 obd=lustre-OST0001-osc-ffff9c7bb744a000 02000000:00000001:3.0:1788205294.430922:0:409030:0:(sec.c:2066:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788205294.430923:0:409030:0:(sec.c:2076:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788205294.430924:0:409030:0:(client.c:845:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788205294.430926:0:409030:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788205294.430927:0:409030:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1788205294.430930:0:409030:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c7bb7412000. 00000100:00000001:3.0:1788205294.430933:0:409030:0:(client.c:925:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205294.430937:0:409030:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788205294.430943:0:409030:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788205294.430945:0:409030:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205294.430949:0:409030:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000087d7c707] to pc [ptlrpcd_rcv+-1] req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788205294.430957:0:409030:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1788205294.430961:0:409030:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (26990) 00000400:00000200:1.0:1788205294.430998:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205294.431001:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205294.431004:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471531:471531:256:4294967295] 192.168.201.118@tcp LPNI seq info [471390:471390:8:4294967295] 00000400:00000200:1.0:1788205294.431009:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.431016:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205294.431021:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.431025:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205294.431029:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205294.431033:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205294.431037:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205294.431040:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205294.431043:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.431045:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205294.431048:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205294.431052:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205294.431055:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205294.431057:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205294.431059:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205294.431063:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431071:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.431074:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431080:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.431082:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205294.431085:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205294.431088:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: using a timeout of 18491 usecs (19 jiffies) 00000400:00000200:1.0:1788205294.431109:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205294.431111:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321653270 00000100:00000001:1.0:1788205294.431113:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321654534 00000100:00000001:1.0:1788205294.431258:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321653270 00000100:00000001:1.0:1788205294.431260:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321654534 00000100:00000001:1.0:1788205294.431262:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205294.431265:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.431267:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205294.431272:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205294/real 1788205294] req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205294.431282:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205294.431283:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205294.431285:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205294.431288:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205294.431289:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205294.431292:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205294.431295:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205294.431303:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431310:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205294.431311:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431313:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.431314:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205294.431317:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431324:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205294.431328:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431334:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205294.431336:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431338:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.431339:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431341:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205294.431343:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205294.431345:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7a91809800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205294.431348:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205294.431350:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205294.431353:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205294.431361:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543535360:192.168.201.118@tcp:38:kworker.0 00000100:00000001:1.0:1788205294.431364:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431366:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.431369:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d39c00 x1875055543535360/t0(0) o38->lustre-MDT0001-mdc-ffff9c7bb744a000@192.168.201.118@tcp:12/10 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431375:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205294.431377:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205294.431379:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205294.431382:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205294.431383:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205294.431384:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7a91809800 refcount=27 obd=lustre-MDT0001-mdc-ffff9c7bb744a000 00000020:00000001:1.0:1788205294.431386:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205294.431387:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412c00. 02000000:00000001:1.0:1788205294.431390:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205294.431391:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205294.431393:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d39c00. 00000100:00000001:1.0:1788205294.431395:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205294.431396:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431398:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431402:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205294.431403:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.431405:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205294.431406:0:256132:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205294.431407:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205294.431411:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431418:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205294.431419:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.431421:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205294.431423:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205294.431425:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205294.431427:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543535488:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205294.431430:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205294.431433:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205294.431435:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.431437:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205294.431498:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205294.431501:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205294.431505:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205294.431507:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205294.431512:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205294.431515:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543535488, portal 4 00000100:00000001:1.0:1788205294.431517:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205294.431518:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643420800 : -109420066130816 : ffff9c7ba7d3b480) 00000100:00000040:1.0:1788205294.431524:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431530:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205294.431532:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205294.431534:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205294.431537:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543535488, offset 0 00000400:00000200:1.0:1788205294.431541:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205294.431545:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.431548:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205294.431551:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205294.431554:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205294.431556:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205294.431558:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205294.431562:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471532:471532:256:4294967295] 192.168.201.118@tcp LPNI seq info [471391:471391:8:4294967295] 00000400:00000200:1.0:1788205294.431566:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.431572:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205294.431576:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.431579:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205294.431583:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205294.431586:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205294.431590:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205294.431592:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205294.431596:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.431598:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205294.431601:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205294.431604:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205294.431607:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205294.431609:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205294.431611:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205294.431614:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431622:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.431625:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431632:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.431633:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205294.431636:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205294.431638:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: using a timeout of 27563 usecs (28 jiffies) 00000400:00000200:1.0:1788205294.431641:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205294.431643:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321653279 00000100:00000001:1.0:1788205294.431645:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321654534 00000100:00000001:1.0:1788205294.431647:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321653279 00000100:00000001:1.0:1788205294.431649:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321654534 00000100:00000001:1.0:1788205294.431651:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205294.431653:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.431655:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205294.431658:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205294/real 1788205294] req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205294.431734:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205294.431736:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205294.431738:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205294.431741:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205294.431742:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205294.431745:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205294.431749:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205294.431757:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431763:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205294.431765:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431766:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.431768:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205294.431771:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431777:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205294.431781:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431788:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205294.431790:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431792:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.431793:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.431794:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205294.431796:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205294.431799:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cf8800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205294.431802:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205294.431804:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205294.431808:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205294.431815:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543535488:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205294.431818:0:256132:0:(client.c:1800:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1788205294.431821:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431827:0:256132:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205294.431829:0:256132:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.431831:0:256132:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788205294.431832:0:256132:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788205294.431834:0:256132:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205294.431837:0:256132:0:(client.c:1906:ptlrpc_send_new_req()) Sending RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543535616:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205294.431840:0:256132:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788205294.431842:0:256132:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788205294.431843:0:256132:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788205294.431845:0:256132:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1788205294.431848:0:256132:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205294.431850:0:256132:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1788205294.431853:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000010:1.0:1788205294.431856:0:256132:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c7b82a18160. 00000400:00000010:1.0:1788205294.431858:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6088. 00000100:00000200:1.0:1788205294.431861:0:256132:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875055543535616, portal 4 00000100:00000001:1.0:1788205294.431863:0:256132:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788205294.431864:0:256132:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446634653643407360 : -109420066144256 : ffff9c7ba7d38000) 00000100:00000040:1.0:1788205294.431869:0:256132:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431876:0:256132:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1788205294.431878:0:256132:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.118@tcp 00000400:00000010:1.0:1788205294.431880:0:256132:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c7b867a6000. 00000100:00000200:1.0:1788205294.431882:0:256132:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1875055543535616, offset 0 00000400:00000200:1.0:1788205294.431886:0:256132:0:(lib-move.c:5129:LNetPut()) LNetPut -> 12345-192.168.201.118@tcp 00000400:00000200:1.0:1788205294.431889:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.431892:0:256132:0:(lib-move.c:2778:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.118@tcp local destination 00000400:00000200:1.0:1788205294.431894:0:256132:0:(lib-move.c:2409:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788205294.431897:0:256132:0:(lib-move.c:1590:lnet_get_best_ni()) selected best_ni 192.168.201.18@tcp 00000400:00000200:1.0:1788205294.431899:0:256132:0:(lib-move.c:1143:lnet_select_peer_ni()) 192.168.201.18@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788205294.431901:0:256132:0:(lib-move.c:1212:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.118@tcp 00000400:00000200:1.0:1788205294.431905:0:256132:0:(lib-move.c:1710:lnet_handle_send()) 192.168.201.18@tcp NI seq info: [471533:471533:256:4294967295] 192.168.201.118@tcp LPNI seq info [471392:471392:8:4294967295] 00000400:00000200:1.0:1788205294.431909:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000400:00000200:1.0:1788205294.431915:0:256132:0:(lib-move.c:1826:lnet_handle_send()) TRACE: 192.168.201.18@tcp(192.168.201.18@tcp:) -> 192.168.201.118@tcp(192.168.201.118@tcp:192.168.201.118@tcp) : PUT try# 0 00000800:00000200:1.0:1788205294.431919:0:256132:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.431922:0:256132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c7b90a97700. 00000800:00000200:1.0:1788205294.431925:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.0:1788205294.431928:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1788205294.431932:0:256132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c7b86a83540] -> 12345-192.168.201.118@tcp (2) 00000800:00000200:1.1:1788205294.431934:0:256132:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.118 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1788205294.431938:0:256132:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.118@tcp 00000800:00000010:1.0:1788205294.431940:0:256132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c7b90a97700. 00000400:00000200:1.0:1788205294.431942:0:256132:0:(lib-msg.c:1127:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1788205294.431946:0:256132:0:(lib-msg.c:900:lnet_health_check()) health check: 192.168.201.18@tcp->192.168.201.118@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1788205294.431948:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6000 00000400:00000010:1.0:1788205294.431950:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6000. 00000100:00000001:1.0:1788205294.431952:0:256132:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1788205294.431955:0:256132:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431963:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.431966:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 0 dl 1788205319 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.431972:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.431973:0:256132:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1788205294.431976:0:256132:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1788205294.431978:0:256132:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: using a timeout of 50132 usecs (51 jiffies) 00000400:00000200:1.0:1788205294.431981:0:256132:0:(api-ni.c:1566:lnet_nid4_cpt_hash()) Match nid 192.168.201.118@tcp to cpt 1 00000100:00000001:1.0:1788205294.431983:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 2 is 4321653302 00000100:00000001:1.0:1788205294.431985:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 2, keep current deadline 4321654534 00000100:00000001:1.0:1788205294.431987:0:256132:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS new deadline estimation for cpu 3 is 4321653302 00000100:00000001:1.0:1788205294.431989:0:256132:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff9c7bb744a000: PM QoS request active for cpu 3, keep current deadline 4321654534 00000100:00000001:1.0:1788205294.431991:0:256132:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788205294.431993:0:256132:0:(client.c:1928:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.431995:0:256132:0:(client.c:2472:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788205294.431998:0:256132:0:(client.c:2490:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788205294/real 1788205294] req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788205294.432006:0:256132:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788205294.432007:0:256132:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1788205294.432009:0:256132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c7b82a18160. 00000400:00000200:1.0:1788205294.432011:0:256132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c7b867a6088 00000400:00000010:1.0:1788205294.432012:0:256132:0:(lib-lnet.h:278:lnet_md_free()) slab-freed 'md': 136 at ffff9c7b867a6088. 00000100:00000001:1.0:1788205294.432014:0:256132:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1788205294.432017:0:256132:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788205294.432025:0:256132:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.432031:0:256132:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788205294.432032:0:256132:0:(client.c:2988:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.432034:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.432035:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788205294.432038:0:256132:0:(client.c:2521:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.432044:0:256132:0:(client.c:2526:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1788205294.432047:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.432054:0:256132:0:(client.c:2111:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788205294.432056:0:256132:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.432057:0:256132:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788205294.432058:0:256132:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.432060:0:256132:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788205294.432061:0:256132:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788205294.432063:0:256132:0:(import.c:65:import_set_state_nolock()) ffff9c7aa3cfd000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788205294.432066:0:256132:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.118@tcp failed (-110) 00000100:00000001:1.0:1788205294.432069:0:256132:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1788205294.432072:0:256132:0:(lustre_net.h:2426:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788205294.432079:0:256132:0:(client.c:2401:ptlrpc_check_set()) Completed RPC req@ffff9c7ba7d38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2d12ff60-d499-4808-9599-5b7284f3b7c2:256132:1875055543535616:192.168.201.118@tcp:8:kworker.0 00000100:00000001:1.0:1788205294.432082:0:256132:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.432084:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.432086:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d3b480 x1875055543535488/t0(0) o8->lustre-OST0000-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.432113:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205294.432539:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205294.432541:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9800. 02000000:00000001:1.0:1788205294.432544:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205294.432546:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205294.432547:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cf8800 refcount=6 obd=lustre-OST0000-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205294.432549:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205294.432551:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7411c00. 02000000:00000001:1.0:1788205294.432553:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205294.432555:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205294.432556:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d3b480. 00000100:00000001:1.0:1788205294.432558:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205294.432559:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.432561:0:256132:0:(client.c:2854:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1788205294.432565:0:256132:0:(client.c:2862:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c7ba7d38000 x1875055543535616/t0(0) o8->lustre-OST0001-osc-ffff9c7bb744a000@192.168.201.118@tcp:28/4 lens 520/544 e 0 to 1 dl 1788205319 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788205294.432571:0:256132:0:(client.c:2780:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788205294.432572:0:256132:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1788205294.432573:0:256132:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c7aab9b9c00. 02000000:00000001:1.0:1788205294.432575:0:256132:0:(sec.c:1914:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788205294.432575:0:256132:0:(genops.c:1239:class_import_put()) Process entered 00000020:00000040:1.0:1788205294.432576:0:256132:0:(genops.c:1245:class_import_put()) import ffff9c7aa3cfd000 refcount=5 obd=lustre-OST0001-osc-ffff9c7bb744a000 00000020:00000001:1.0:1788205294.432578:0:256132:0:(genops.c:1252:class_import_put()) Process leaving 02000000:00000010:1.0:1788205294.432579:0:256132:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c7bb7412000. 02000000:00000001:1.0:1788205294.432581:0:256132:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788205294.432582:0:256132:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1788205294.432583:0:256132:0:(client.c:599:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c7ba7d38000. 00000100:00000001:1.0:1788205294.432586:0:256132:0:(client.c:2836:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788205294.432587:0:256132:0:(client.c:2895:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.432588:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205294.432593:0:256132:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205294.432594:0:256132:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205294.432595:0:256132:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788205294.432597:0:256132:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260237:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205295.260244:0:417027:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260249:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205295.260252:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205295.260255:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205295.260258:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260261:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205295.260263:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260265:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260267:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260270:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205295.260272:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205295.260273:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205295.260274:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260276:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205295.260277:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260279:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260280:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260283:0:417027:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788205295.260285:0:417027:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788205295.260286:0:417027:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788205295.260288:0:417027:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260289:0:417027:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788205295.260290:0:417027:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260292:0:417027:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788205295.260293:0:417027:0:(ldlm_pool.c:498:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788205295.261725:0:404359:0:(genops.c:1924:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788205295.261731:0:404359:0:(genops.c:1938:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788205295.261735:0:404359:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788205295.261802:0:395635:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1788205295.261807:0:395635:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788205295.261809:0:395635:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788205295.261812:0:395635:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7a93ed9c00 going to sleep for 0 seconds 00000100:00000001:3.0:1788205295.261816:0:395635:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788205295.261819:0:395635:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205295.261822:0:395635:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788205295.261824:0:395635:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788205295.261827:0:395635:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788205295.261891:0:395636:0:(client.c:2554:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788205295.261894:0:395636:0:(client.c:2626:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788205295.261895:0:395636:0:(client.c:2654:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788205295.261897:0:395636:0:(client.c:2702:ptlrpc_set_wait()) set ffff9c7ab12933c0 going to sleep for 0 seconds 00000100:00000001:1.0:1788205295.261899:0:395636:0:(client.c:1977:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788205295.261901:0:395636:0:(client.c:2980:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205295.261904:0:395636:0:(client.c:1364:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788205295.261906:0:395636:0:(client.c:1423:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788205295.261907:0:395636:0:(client.c:2452:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1788205295.772587:0:409289:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af220 (cpu 0) has reached its deadline 4321654534, at 4321654592 00000100:00000010:2.0:1788205295.772605:0:409289:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab1647300. 00000100:00000040:3.0:1788205295.773768:0:417298:0:(connection.c:75:cpu_latency_work()) work item of ffff9c7ab56af330 (cpu 0) has reached its deadline 4321654534, at 4321654593 00000100:00000010:3.0:1788205295.773779:0:417298:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c7ab16479c0. Debug log: 138671 lines, 138671 kept, 0 dropped, 0 bad.